Coverage for /root/GitHubProjects/impacket/impacket/ese.py : 67%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Impacket - Collection of Python classes for working with network protocols.
2#
3# SECUREAUTH LABS. Copyright (C) 2018 SecureAuth Corporation. All rights reserved.
4#
5# This software is provided under a slightly modified version
6# of the Apache Software License. See the accompanying LICENSE file
7# for more information.
8#
9# Description:
10# Microsoft Extensive Storage Engine parser, just focused on trying
11# to parse NTDS.dit files (not meant as a full parser, although it might work)
12#
13# Author:
14# Alberto Solino (@agsolino)
15#
16# Reference for:
17# Structure
18#
19# Excellent reference done by Joachim Metz
20# - http://forensic-proof.com/wp-content/uploads/2011/07/Extensible-Storage-Engine-ESE-Database-File-EDB-format.pdf
21#
22# ToDo:
23# [ ] Parse multi-values properly
24# [ ] Support long values properly
25#
27from __future__ import division
28from __future__ import print_function
29from impacket import LOG
30try:
31 from collections import OrderedDict
32except:
33 try:
34 from ordereddict.ordereddict import OrderedDict
35 except:
36 from ordereddict import OrderedDict
37from impacket.structure import Structure, hexdump
38from struct import unpack
39from binascii import hexlify
40from six import b
42# Constants
44FILE_TYPE_DATABASE = 0
45FILE_TYPE_STREAMING_FILE = 1
47# Database state
48JET_dbstateJustCreated = 1
49JET_dbstateDirtyShutdown = 2
50JET_dbstateCleanShutdown = 3
51JET_dbstateBeingConverted = 4
52JET_dbstateForceDetach = 5
54# Page Flags
55FLAGS_ROOT = 1
56FLAGS_LEAF = 2
57FLAGS_PARENT = 4
58FLAGS_EMPTY = 8
59FLAGS_SPACE_TREE = 0x20
60FLAGS_INDEX = 0x40
61FLAGS_LONG_VALUE = 0x80
62FLAGS_NEW_FORMAT = 0x2000
63FLAGS_NEW_CHECKSUM = 0x2000
65# Tag Flags
66TAG_UNKNOWN = 0x1
67TAG_DEFUNCT = 0x2
68TAG_COMMON = 0x4
70# Fixed Page Numbers
71DATABASE_PAGE_NUMBER = 1
72CATALOG_PAGE_NUMBER = 4
73CATALOG_BACKUP_PAGE_NUMBER = 24
75# Fixed FatherDataPages
76DATABASE_FDP = 1
77CATALOG_FDP = 2
78CATALOG_BACKUP_FDP = 3
80# Catalog Types
81CATALOG_TYPE_TABLE = 1
82CATALOG_TYPE_COLUMN = 2
83CATALOG_TYPE_INDEX = 3
84CATALOG_TYPE_LONG_VALUE = 4
85CATALOG_TYPE_CALLBACK = 5
87# Column Types
88JET_coltypNil = 0
89JET_coltypBit = 1
90JET_coltypUnsignedByte = 2
91JET_coltypShort = 3
92JET_coltypLong = 4
93JET_coltypCurrency = 5
94JET_coltypIEEESingle = 6
95JET_coltypIEEEDouble = 7
96JET_coltypDateTime = 8
97JET_coltypBinary = 9
98JET_coltypText = 10
99JET_coltypLongBinary = 11
100JET_coltypLongText = 12
101JET_coltypSLV = 13
102JET_coltypUnsignedLong = 14
103JET_coltypLongLong = 15
104JET_coltypGUID = 16
105JET_coltypUnsignedShort= 17
106JET_coltypMax = 18
108ColumnTypeToName = {
109 JET_coltypNil : 'NULL',
110 JET_coltypBit : 'Boolean',
111 JET_coltypUnsignedByte : 'Signed byte',
112 JET_coltypShort : 'Signed short',
113 JET_coltypLong : 'Signed long',
114 JET_coltypCurrency : 'Currency',
115 JET_coltypIEEESingle : 'Single precision FP',
116 JET_coltypIEEEDouble : 'Double precision FP',
117 JET_coltypDateTime : 'DateTime',
118 JET_coltypBinary : 'Binary',
119 JET_coltypText : 'Text',
120 JET_coltypLongBinary : 'Long Binary',
121 JET_coltypLongText : 'Long Text',
122 JET_coltypSLV : 'Obsolete',
123 JET_coltypUnsignedLong : 'Unsigned long',
124 JET_coltypLongLong : 'Long long',
125 JET_coltypGUID : 'GUID',
126 JET_coltypUnsignedShort: 'Unsigned short',
127 JET_coltypMax : 'Max',
128}
130ColumnTypeSize = {
131 JET_coltypNil : None,
132 JET_coltypBit : (1,'B'),
133 JET_coltypUnsignedByte : (1,'B'),
134 JET_coltypShort : (2,'<h'),
135 JET_coltypLong : (4,'<l'),
136 JET_coltypCurrency : (8,'<Q'),
137 JET_coltypIEEESingle : (4,'<f'),
138 JET_coltypIEEEDouble : (8,'<d'),
139 JET_coltypDateTime : (8,'<Q'),
140 JET_coltypBinary : None,
141 JET_coltypText : None,
142 JET_coltypLongBinary : None,
143 JET_coltypLongText : None,
144 JET_coltypSLV : None,
145 JET_coltypUnsignedLong : (4,'<L'),
146 JET_coltypLongLong : (8,'<Q'),
147 JET_coltypGUID : (16,'16s'),
148 JET_coltypUnsignedShort: (2,'<H'),
149 JET_coltypMax : None,
150}
152# Tagged Data Type Flags
153TAGGED_DATA_TYPE_VARIABLE_SIZE = 1
154TAGGED_DATA_TYPE_COMPRESSED = 2
155TAGGED_DATA_TYPE_STORED = 4
156TAGGED_DATA_TYPE_MULTI_VALUE = 8
157TAGGED_DATA_TYPE_WHO_KNOWS = 10
159# Code pages
160CODEPAGE_UNICODE = 1200
161CODEPAGE_ASCII = 20127
162CODEPAGE_WESTERN = 1252
164StringCodePages = {
165 CODEPAGE_UNICODE : 'utf-16le',
166 CODEPAGE_ASCII : 'ascii',
167 CODEPAGE_WESTERN : 'cp1252',
168}
170# Structures
172TABLE_CURSOR = {
173 'TableData' : b'',
174 'FatherDataPageNumber': 0,
175 'CurrentPageData' : b'',
176 'CurrentTag' : 0,
177}
179class ESENT_JET_SIGNATURE(Structure):
180 structure = (
181 ('Random','<L=0'),
182 ('CreationTime','<Q=0'),
183 ('NetBiosName','16s=b""'),
184 )
186class ESENT_DB_HEADER(Structure):
187 structure = (
188 ('CheckSum','<L=0'),
189 ('Signature','"\xef\xcd\xab\x89'),
190 ('Version','<L=0'),
191 ('FileType','<L=0'),
192 ('DBTime','<Q=0'),
193 ('DBSignature',':',ESENT_JET_SIGNATURE),
194 ('DBState','<L=0'),
195 ('ConsistentPosition','<Q=0'),
196 ('ConsistentTime','<Q=0'),
197 ('AttachTime','<Q=0'),
198 ('AttachPosition','<Q=0'),
199 ('DetachTime','<Q=0'),
200 ('DetachPosition','<Q=0'),
201 ('LogSignature',':',ESENT_JET_SIGNATURE),
202 ('Unknown','<L=0'),
203 ('PreviousBackup','24s=b""'),
204 ('PreviousIncBackup','24s=b""'),
205 ('CurrentFullBackup','24s=b""'),
206 ('ShadowingDisables','<L=0'),
207 ('LastObjectID','<L=0'),
208 ('WindowsMajorVersion','<L=0'),
209 ('WindowsMinorVersion','<L=0'),
210 ('WindowsBuildNumber','<L=0'),
211 ('WindowsServicePackNumber','<L=0'),
212 ('FileFormatRevision','<L=0'),
213 ('PageSize','<L=0'),
214 ('RepairCount','<L=0'),
215 ('RepairTime','<Q=0'),
216 ('Unknown2','28s=b""'),
217 ('ScrubTime','<Q=0'),
218 ('RequiredLog','<Q=0'),
219 ('UpgradeExchangeFormat','<L=0'),
220 ('UpgradeFreePages','<L=0'),
221 ('UpgradeSpaceMapPages','<L=0'),
222 ('CurrentShadowBackup','24s=b""'),
223 ('CreationFileFormatVersion','<L=0'),
224 ('CreationFileFormatRevision','<L=0'),
225 ('Unknown3','16s=b""'),
226 ('OldRepairCount','<L=0'),
227 ('ECCCount','<L=0'),
228 ('LastECCTime','<Q=0'),
229 ('OldECCFixSuccessCount','<L=0'),
230 ('ECCFixErrorCount','<L=0'),
231 ('LastECCFixErrorTime','<Q=0'),
232 ('OldECCFixErrorCount','<L=0'),
233 ('BadCheckSumErrorCount','<L=0'),
234 ('LastBadCheckSumTime','<Q=0'),
235 ('OldCheckSumErrorCount','<L=0'),
236 ('CommittedLog','<L=0'),
237 ('PreviousShadowCopy','24s=b""'),
238 ('PreviousDifferentialBackup','24s=b""'),
239 ('Unknown4','40s=b""'),
240 ('NLSMajorVersion','<L=0'),
241 ('NLSMinorVersion','<L=0'),
242 ('Unknown5','148s=b""'),
243 ('UnknownFlags','<L=0'),
244 )
246class ESENT_PAGE_HEADER(Structure):
247 structure_2003_SP0 = (
248 ('CheckSum','<L=0'),
249 ('PageNumber','<L=0'),
250 )
251 structure_0x620_0x0b = (
252 ('CheckSum','<L=0'),
253 ('ECCCheckSum','<L=0'),
254 )
255 structure_win7 = (
256 ('CheckSum','<Q=0'),
257 )
258 common = (
259 ('LastModificationTime','<Q=0'),
260 ('PreviousPageNumber','<L=0'),
261 ('NextPageNumber','<L=0'),
262 ('FatherDataPage','<L=0'),
263 ('AvailableDataSize','<H=0'),
264 ('AvailableUncommittedDataSize','<H=0'),
265 ('FirstAvailableDataOffset','<H=0'),
266 ('FirstAvailablePageTag','<H=0'),
267 ('PageFlags','<L=0'),
268 )
269 extended_win7 = (
270 ('ExtendedCheckSum1','<Q=0'),
271 ('ExtendedCheckSum2','<Q=0'),
272 ('ExtendedCheckSum3','<Q=0'),
273 ('PageNumber','<Q=0'),
274 ('Unknown','<Q=0'),
275 )
276 def __init__(self, version, revision, pageSize=8192, data=None):
277 if (version < 0x620) or (version == 0x620 and revision < 0x0b): 277 ↛ 279line 277 didn't jump to line 279, because the condition on line 277 was never true
278 # For sure the old format
279 self.structure = self.structure_2003_SP0 + self.common
280 elif version == 0x620 and revision < 0x11: 280 ↛ 282line 280 didn't jump to line 282, because the condition on line 280 was never true
281 # Exchange 2003 SP1 and Windows Vista and later
282 self.structure = self.structure_0x620_0x0b + self.common
283 else:
284 # Windows 7 and later
285 self.structure = self.structure_win7 + self.common
286 if pageSize > 8192: 286 ↛ 287line 286 didn't jump to line 287, because the condition on line 286 was never true
287 self.structure += self.extended_win7
289 Structure.__init__(self,data)
291class ESENT_ROOT_HEADER(Structure):
292 structure = (
293 ('InitialNumberOfPages','<L=0'),
294 ('ParentFatherDataPage','<L=0'),
295 ('ExtentSpace','<L=0'),
296 ('SpaceTreePageNumber','<L=0'),
297 )
299class ESENT_BRANCH_HEADER(Structure):
300 structure = (
301 ('CommonPageKey',':'),
302 )
304class ESENT_BRANCH_ENTRY(Structure):
305 common = (
306 ('CommonPageKeySize','<H=0'),
307 )
308 structure = (
309 ('LocalPageKeySize','<H=0'),
310 ('_LocalPageKey','_-LocalPageKey','self["LocalPageKeySize"]'),
311 ('LocalPageKey',':'),
312 ('ChildPageNumber','<L=0'),
313 )
314 def __init__(self, flags, data=None):
315 if flags & TAG_COMMON > 0:
316 # Include the common header
317 self.structure = self.common + self.structure
318 Structure.__init__(self,data)
320class ESENT_LEAF_HEADER(Structure):
321 structure = (
322 ('CommonPageKey',':'),
323 )
325class ESENT_LEAF_ENTRY(Structure):
326 common = (
327 ('CommonPageKeySize','<H=0'),
328 )
329 structure = (
330 ('LocalPageKeySize','<H=0'),
331 ('_LocalPageKey','_-LocalPageKey','self["LocalPageKeySize"]'),
332 ('LocalPageKey',':'),
333 ('EntryData',':'),
334 )
335 def __init__(self, flags, data=None):
336 if flags & TAG_COMMON > 0:
337 # Include the common header
338 self.structure = self.common + self.structure
339 Structure.__init__(self,data)
341class ESENT_SPACE_TREE_HEADER(Structure):
342 structure = (
343 ('Unknown','<Q=0'),
344 )
346class ESENT_SPACE_TREE_ENTRY(Structure):
347 structure = (
348 ('PageKeySize','<H=0'),
349 ('LastPageNumber','<L=0'),
350 ('NumberOfPages','<L=0'),
351 )
353class ESENT_INDEX_ENTRY(Structure):
354 structure = (
355 ('RecordPageKey',':'),
356 )
358class ESENT_DATA_DEFINITION_HEADER(Structure):
359 structure = (
360 ('LastFixedSize','<B=0'),
361 ('LastVariableDataType','<B=0'),
362 ('VariableSizeOffset','<H=0'),
363 )
365class ESENT_CATALOG_DATA_DEFINITION_ENTRY(Structure):
366 fixed = (
367 ('FatherDataPageID','<L=0'),
368 ('Type','<H=0'),
369 ('Identifier','<L=0'),
370 )
372 column_stuff = (
373 ('ColumnType','<L=0'),
374 ('SpaceUsage','<L=0'),
375 ('ColumnFlags','<L=0'),
376 ('CodePage','<L=0'),
377 )
379 other = (
380 ('FatherDataPageNumber','<L=0'),
381 )
383 table_stuff = (
384 ('SpaceUsage','<L=0'),
385# ('TableFlags','<L=0'),
386# ('InitialNumberOfPages','<L=0'),
387 )
389 index_stuff = (
390 ('SpaceUsage','<L=0'),
391 ('IndexFlags','<L=0'),
392 ('Locale','<L=0'),
393 )
395 lv_stuff = (
396 ('SpaceUsage','<L=0'),
397# ('LVFlags','<L=0'),
398# ('InitialNumberOfPages','<L=0'),
399 )
400 common = (
401# ('RootFlag','<B=0'),
402# ('RecordOffset','<H=0'),
403# ('LCMapFlags','<L=0'),
404# ('KeyMost','<H=0'),
405 ('Trailing',':'),
406 )
408 def __init__(self,data):
409 # Depending on the type of data we'll end up building a different struct
410 dataType = unpack('<H', data[4:][:2])[0]
411 self.structure = self.fixed
413 if dataType == CATALOG_TYPE_TABLE:
414 self.structure += self.other + self.table_stuff
415 elif dataType == CATALOG_TYPE_COLUMN:
416 self.structure += self.column_stuff
417 elif dataType == CATALOG_TYPE_INDEX:
418 self.structure += self.other + self.index_stuff
419 elif dataType == CATALOG_TYPE_LONG_VALUE: 419 ↛ 421line 419 didn't jump to line 421, because the condition on line 419 was never false
420 self.structure += self.other + self.lv_stuff
421 elif dataType == CATALOG_TYPE_CALLBACK:
422 raise Exception('CallBack types not supported!')
423 else:
424 LOG.error('Unknown catalog type 0x%x' % dataType)
425 self.structure = ()
426 Structure.__init__(self,data)
428 self.structure += self.common
430 Structure.__init__(self,data)
433#def pretty_print(x):
434# if x in '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ ':
435# return x
436# else:
437# return '.'
438#
439#def hexdump(data):
440# x=str(data)
441# strLen = len(x)
442# i = 0
443# while i < strLen:
444# print "%04x " % i,
445# for j in range(16):
446# if i+j < strLen:
447# print "%02X" % ord(x[i+j]),
448#
449# else:
450# print " ",
451# if j%16 == 7:
452# print "",
453# print " ",
454# print ''.join(pretty_print(x) for x in x[i:i+16] )
455# i += 16
457def getUnixTime(t):
458 t -= 116444736000000000
459 t //= 10000000
460 return t
462class ESENT_PAGE:
463 def __init__(self, db, data=None):
464 self.__DBHeader = db
465 self.data = data
466 self.record = None
467 if data is not None: 467 ↛ exitline 467 didn't return from function '__init__', because the condition on line 467 was never false
468 self.record = ESENT_PAGE_HEADER(self.__DBHeader['Version'], self.__DBHeader['FileFormatRevision'], self.__DBHeader['PageSize'], data)
470 def printFlags(self):
471 flags = self.record['PageFlags']
472 if flags & FLAGS_EMPTY:
473 print("\tEmpty")
474 if flags & FLAGS_INDEX:
475 print("\tIndex")
476 if flags & FLAGS_LEAF:
477 print("\tLeaf")
478 else:
479 print("\tBranch")
480 if flags & FLAGS_LONG_VALUE:
481 print("\tLong Value")
482 if flags & FLAGS_NEW_CHECKSUM:
483 print("\tNew Checksum")
484 if flags & FLAGS_NEW_FORMAT:
485 print("\tNew Format")
486 if flags & FLAGS_PARENT:
487 print("\tParent")
488 if flags & FLAGS_ROOT:
489 print("\tRoot")
490 if flags & FLAGS_SPACE_TREE:
491 print("\tSpace Tree")
493 def dump(self):
494 baseOffset = len(self.record)
495 self.record.dump()
496 tags = self.data[-4*self.record['FirstAvailablePageTag']:]
498 print("FLAGS: ")
499 self.printFlags()
501 print()
503 for i in range(self.record['FirstAvailablePageTag']):
504 tag = tags[-4:]
505 if self.__DBHeader['Version'] == 0x620 and self.__DBHeader['FileFormatRevision'] > 11 and self.__DBHeader['PageSize'] > 8192:
506 valueSize = unpack('<H', tag[:2])[0] & 0x7fff
507 valueOffset = unpack('<H',tag[2:])[0] & 0x7fff
508 hexdump((self.data[baseOffset+valueOffset:][:6]))
509 pageFlags = ord(self.data[baseOffset+valueOffset:][1]) >> 5
510 #print "TAG FLAG: 0x%x " % (unpack('<L', self.data[baseOffset+valueOffset:][:4]) ) >> 5
511 #print "TAG FLAG: 0x " , ord(self.data[baseOffset+valueOffset:][0])
512 else:
513 valueSize = unpack('<H', tag[:2])[0] & 0x1fff
514 pageFlags = (unpack('<H', tag[2:])[0] & 0xe000) >> 13
515 valueOffset = unpack('<H',tag[2:])[0] & 0x1fff
517 print("TAG %-8d offset:0x%-6x flags:0x%-4x valueSize:0x%x" % (i,valueOffset,pageFlags,valueSize))
518 #hexdump(self.getTag(i)[1])
519 tags = tags[:-4]
521 if self.record['PageFlags'] & FLAGS_ROOT > 0:
522 rootHeader = ESENT_ROOT_HEADER(self.getTag(0)[1])
523 rootHeader.dump()
524 elif self.record['PageFlags'] & FLAGS_LEAF == 0:
525 # Branch Header
526 flags, data = self.getTag(0)
527 branchHeader = ESENT_BRANCH_HEADER(data)
528 branchHeader.dump()
529 else:
530 # Leaf Header
531 flags, data = self.getTag(0)
532 if self.record['PageFlags'] & FLAGS_SPACE_TREE > 0:
533 # Space Tree
534 spaceTreeHeader = ESENT_SPACE_TREE_HEADER(data)
535 spaceTreeHeader.dump()
536 else:
537 leafHeader = ESENT_LEAF_HEADER(data)
538 leafHeader.dump()
540 # Print the leaf/branch tags
541 for tagNum in range(1,self.record['FirstAvailablePageTag']):
542 flags, data = self.getTag(tagNum)
543 if self.record['PageFlags'] & FLAGS_LEAF == 0:
544 # Branch page
545 branchEntry = ESENT_BRANCH_ENTRY(flags, data)
546 branchEntry.dump()
547 elif self.record['PageFlags'] & FLAGS_LEAF > 0:
548 # Leaf page
549 if self.record['PageFlags'] & FLAGS_SPACE_TREE > 0:
550 # Space Tree
551 spaceTreeEntry = ESENT_SPACE_TREE_ENTRY(data)
552 #spaceTreeEntry.dump()
554 elif self.record['PageFlags'] & FLAGS_INDEX > 0:
555 # Index Entry
556 indexEntry = ESENT_INDEX_ENTRY(data)
557 #indexEntry.dump()
558 elif self.record['PageFlags'] & FLAGS_LONG_VALUE > 0:
559 # Long Page Value
560 raise Exception('Long value still not supported')
561 else:
562 # Table Value
563 leafEntry = ESENT_LEAF_ENTRY(flags, data)
564 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(leafEntry['EntryData'])
565 dataDefinitionHeader.dump()
566 catalogEntry = ESENT_CATALOG_DATA_DEFINITION_ENTRY(leafEntry['EntryData'][len(dataDefinitionHeader):])
567 catalogEntry.dump()
568 hexdump(leafEntry['EntryData'])
570 def getTag(self, tagNum):
571 if self.record['FirstAvailablePageTag'] < tagNum: 571 ↛ 572line 571 didn't jump to line 572, because the condition on line 571 was never true
572 raise Exception('Trying to grab an unknown tag 0x%x' % tagNum)
574 tags = self.data[-4*self.record['FirstAvailablePageTag']:]
575 baseOffset = len(self.record)
576 for i in range(tagNum):
577 tags = tags[:-4]
579 tag = tags[-4:]
581 if self.__DBHeader['Version'] == 0x620 and self.__DBHeader['FileFormatRevision'] >= 17 and self.__DBHeader['PageSize'] > 8192: 581 ↛ 582line 581 didn't jump to line 582, because the condition on line 581 was never true
582 valueSize = unpack('<H', tag[:2])[0] & 0x7fff
583 valueOffset = unpack('<H',tag[2:])[0] & 0x7fff
584 tmpData = list(self.data[baseOffset+valueOffset:][:valueSize])
585 pageFlags = ord(tmpData[1]) >> 5
586 tmpData[1] = chr(ord(tmpData[1:2]) & 0x1f)
587 tagData = "".join(tmpData)
588 else:
589 valueSize = unpack('<H', tag[:2])[0] & 0x1fff
590 pageFlags = (unpack('<H', tag[2:])[0] & 0xe000) >> 13
591 valueOffset = unpack('<H',tag[2:])[0] & 0x1fff
592 tagData = self.data[baseOffset+valueOffset:][:valueSize]
594 #return pageFlags, self.data[baseOffset+valueOffset:][:valueSize]
595 return pageFlags, tagData
597class ESENT_DB:
598 def __init__(self, fileName, pageSize = 8192, isRemote = False):
599 self.__fileName = fileName
600 self.__pageSize = pageSize
601 self.__DB = None
602 self.__DBHeader = None
603 self.__totalPages = None
604 self.__tables = OrderedDict()
605 self.__currentTable = None
606 self.__isRemote = isRemote
607 self.mountDB()
609 def mountDB(self):
610 LOG.debug("Mounting DB...")
611 if self.__isRemote is True: 611 ↛ 615line 611 didn't jump to line 615, because the condition on line 611 was never false
612 self.__DB = self.__fileName
613 self.__DB.open()
614 else:
615 self.__DB = open(self.__fileName,"rb")
616 mainHeader = self.getPage(-1)
617 self.__DBHeader = ESENT_DB_HEADER(mainHeader)
618 self.__pageSize = self.__DBHeader['PageSize']
619 self.__DB.seek(0,2)
620 self.__totalPages = (self.__DB.tell() // self.__pageSize) -2
621 LOG.debug("Database Version:0x%x, Revision:0x%x"% (self.__DBHeader['Version'], self.__DBHeader['FileFormatRevision']))
622 LOG.debug("Page Size: %d" % self.__pageSize)
623 LOG.debug("Total Pages in file: %d" % self.__totalPages)
624 self.parseCatalog(CATALOG_PAGE_NUMBER)
626 def printCatalog(self):
627 indent = ' '
629 print("Database version: 0x%x, 0x%x" % (self.__DBHeader['Version'], self.__DBHeader['FileFormatRevision'] ))
630 print("Page size: %d " % self.__pageSize)
631 print("Number of pages: %d" % self.__totalPages)
632 print()
633 print("Catalog for %s" % self.__fileName)
634 for table in list(self.__tables.keys()):
635 print("[%s]" % table.decode('utf8'))
636 print("%sColumns " % indent)
637 for column in list(self.__tables[table]['Columns'].keys()):
638 record = self.__tables[table]['Columns'][column]['Record']
639 print("%s%-5d%-30s%s" % (indent*2, record['Identifier'], column.decode('utf-8'),ColumnTypeToName[record['ColumnType']]))
640 print("%sIndexes"% indent)
641 for index in list(self.__tables[table]['Indexes'].keys()):
642 print("%s%s" % (indent*2, index.decode('utf-8')))
643 print("")
645 def __addItem(self, entry):
646 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData'])
647 catalogEntry = ESENT_CATALOG_DATA_DEFINITION_ENTRY(entry['EntryData'][len(dataDefinitionHeader):])
648 itemName = self.__parseItemName(entry)
650 if catalogEntry['Type'] == CATALOG_TYPE_TABLE:
651 self.__tables[itemName] = OrderedDict()
652 self.__tables[itemName]['TableEntry'] = entry
653 self.__tables[itemName]['Columns'] = OrderedDict()
654 self.__tables[itemName]['Indexes'] = OrderedDict()
655 self.__tables[itemName]['LongValues'] = OrderedDict()
656 self.__currentTable = itemName
657 elif catalogEntry['Type'] == CATALOG_TYPE_COLUMN:
658 self.__tables[self.__currentTable]['Columns'][itemName] = entry
659 self.__tables[self.__currentTable]['Columns'][itemName]['Header'] = dataDefinitionHeader
660 self.__tables[self.__currentTable]['Columns'][itemName]['Record'] = catalogEntry
661 elif catalogEntry['Type'] == CATALOG_TYPE_INDEX:
662 self.__tables[self.__currentTable]['Indexes'][itemName] = entry
663 elif catalogEntry['Type'] == CATALOG_TYPE_LONG_VALUE: 663 ↛ 666line 663 didn't jump to line 666, because the condition on line 663 was never false
664 self.__addLongValue(entry)
665 else:
666 raise Exception('Unknown type 0x%x' % catalogEntry['Type'])
668 def __parseItemName(self,entry):
669 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData'])
671 if dataDefinitionHeader['LastVariableDataType'] > 127: 671 ↛ 674line 671 didn't jump to line 674, because the condition on line 671 was never false
672 numEntries = dataDefinitionHeader['LastVariableDataType'] - 127
673 else:
674 numEntries = dataDefinitionHeader['LastVariableDataType']
676 itemLen = unpack('<H',entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][:2])[0]
677 itemName = entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][2*numEntries:][:itemLen]
678 return itemName
680 def __addLongValue(self, entry):
681 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData'])
682 lvLen = unpack('<H',entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][:2])[0]
683 lvName = entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][7:][:lvLen]
684 self.__tables[self.__currentTable]['LongValues'][lvName] = entry
686 def parsePage(self, page):
687 # Print the leaf/branch tags
688 for tagNum in range(1,page.record['FirstAvailablePageTag']):
689 flags, data = page.getTag(tagNum)
690 if page.record['PageFlags'] & FLAGS_LEAF > 0:
691 # Leaf page
692 if page.record['PageFlags'] & FLAGS_SPACE_TREE > 0: 692 ↛ 693line 692 didn't jump to line 693, because the condition on line 692 was never true
693 pass
694 elif page.record['PageFlags'] & FLAGS_INDEX > 0: 694 ↛ 695line 694 didn't jump to line 695, because the condition on line 694 was never true
695 pass
696 elif page.record['PageFlags'] & FLAGS_LONG_VALUE > 0: 696 ↛ 697line 696 didn't jump to line 697, because the condition on line 696 was never true
697 pass
698 else:
699 # Table Value
700 leafEntry = ESENT_LEAF_ENTRY(flags, data)
701 self.__addItem(leafEntry)
703 def parseCatalog(self, pageNum):
704 # Parse all the pages starting at pageNum and commit table data
705 page = self.getPage(pageNum)
706 self.parsePage(page)
708 for i in range(1, page.record['FirstAvailablePageTag']):
709 flags, data = page.getTag(i)
710 if page.record['PageFlags'] & FLAGS_LEAF == 0:
711 # Branch page
712 branchEntry = ESENT_BRANCH_ENTRY(flags, data)
713 self.parseCatalog(branchEntry['ChildPageNumber'])
716 def readHeader(self):
717 LOG.debug("Reading Boot Sector for %s" % self.__volumeName)
719 def getPage(self, pageNum):
720 LOG.debug("Trying to fetch page %d (0x%x)" % (pageNum, (pageNum+1)*self.__pageSize))
721 self.__DB.seek((pageNum+1)*self.__pageSize, 0)
722 data = self.__DB.read(self.__pageSize)
723 while len(data) < self.__pageSize: 723 ↛ 724line 723 didn't jump to line 724, because the condition on line 723 was never true
724 remaining = self.__pageSize - len(data)
725 data += self.__DB.read(remaining)
726 # Special case for the first page
727 if pageNum <= 0:
728 return data
729 else:
730 return ESENT_PAGE(self.__DBHeader, data)
732 def close(self):
733 self.__DB.close()
735 def openTable(self, tableName):
736 # Returns a cursos for later use
738 if isinstance(tableName, bytes) is not True: 738 ↛ 741line 738 didn't jump to line 741, because the condition on line 738 was never false
739 tableName = b(tableName)
741 if tableName in self.__tables: 741 ↛ 772line 741 didn't jump to line 772, because the condition on line 741 was never false
742 entry = self.__tables[tableName]['TableEntry']
743 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData'])
744 catalogEntry = ESENT_CATALOG_DATA_DEFINITION_ENTRY(entry['EntryData'][len(dataDefinitionHeader):])
746 # Let's position the cursor at the leaf levels for fast reading
747 pageNum = catalogEntry['FatherDataPageNumber']
748 done = False
749 while done is False:
750 page = self.getPage(pageNum)
751 if page.record['FirstAvailablePageTag'] <= 1: 751 ↛ 753line 751 didn't jump to line 753, because the condition on line 751 was never true
752 # There are no records
753 done = True
754 for i in range(1, page.record['FirstAvailablePageTag']): 754 ↛ 749line 754 didn't jump to line 749, because the loop on line 754 didn't complete
755 flags, data = page.getTag(i)
756 if page.record['PageFlags'] & FLAGS_LEAF == 0:
757 # Branch page, move on to the next page
758 branchEntry = ESENT_BRANCH_ENTRY(flags, data)
759 pageNum = branchEntry['ChildPageNumber']
760 break
761 else:
762 done = True
763 break
765 cursor = TABLE_CURSOR
766 cursor['TableData'] = self.__tables[tableName]
767 cursor['FatherDataPageNumber'] = catalogEntry['FatherDataPageNumber']
768 cursor['CurrentPageData'] = page
769 cursor['CurrentTag'] = 0
770 return cursor
771 else:
772 return None
774 def __getNextTag(self, cursor):
775 page = cursor['CurrentPageData']
777 if cursor['CurrentTag'] >= page.record['FirstAvailablePageTag']:
778 # No more data in this page, chau
779 return None
781 flags, data = page.getTag(cursor['CurrentTag'])
782 if page.record['PageFlags'] & FLAGS_LEAF > 0: 782 ↛ 795line 782 didn't jump to line 795, because the condition on line 782 was never false
783 # Leaf page
784 if page.record['PageFlags'] & FLAGS_SPACE_TREE > 0: 784 ↛ 785line 784 didn't jump to line 785, because the condition on line 784 was never true
785 raise Exception('FLAGS_SPACE_TREE > 0')
786 elif page.record['PageFlags'] & FLAGS_INDEX > 0: 786 ↛ 787line 786 didn't jump to line 787, because the condition on line 786 was never true
787 raise Exception('FLAGS_INDEX > 0')
788 elif page.record['PageFlags'] & FLAGS_LONG_VALUE > 0: 788 ↛ 789line 788 didn't jump to line 789, because the condition on line 788 was never true
789 raise Exception('FLAGS_LONG_VALUE > 0')
790 else:
791 # Table Value
792 leafEntry = ESENT_LEAF_ENTRY(flags, data)
793 return leafEntry
795 return None
797 def getNextRow(self, cursor, filter_tables = None):
798 cursor['CurrentTag'] += 1
800 tag = self.__getNextTag(cursor)
801 #hexdump(tag)
803 if tag is None:
804 # No more tags in this page, search for the next one on the right
805 page = cursor['CurrentPageData']
806 if page.record['NextPageNumber'] == 0:
807 # No more pages, chau
808 return None
809 else:
810 cursor['CurrentPageData'] = self.getPage(page.record['NextPageNumber'])
811 cursor['CurrentTag'] = 0
812 return self.getNextRow(cursor, filter_tables = filter_tables)
813 else:
814 return self.__tagToRecord(cursor, tag['EntryData'], filter_tables = filter_tables)
816 def __tagToRecord(self, cursor, tag, filter_tables = None):
817 # So my brain doesn't forget, the data record is composed of:
818 # Header
819 # Fixed Size Data (ID < 127)
820 # The easiest to parse. Their size is fixed in the record. You can get its size
821 # from the Column Record, field SpaceUsage
822 # Variable Size Data (127 < ID < 255)
823 # At VariableSizeOffset you get an array of two bytes per variable entry, pointing
824 # to the length of the value. Values start at:
825 # numEntries = LastVariableDataType - 127
826 # VariableSizeOffset + numEntries * 2 (bytes)
827 # Tagged Data ( > 255 )
828 # After the Variable Size Value, there's more data for the tagged values.
829 # Right at the beginning there's another array (taggedItems), pointing to the
830 # values, size.
831 #
832 # The interesting thing about this DB records is there's no need for all the columns to be there, hence
833 # saving space. That's why I got over all the columns, and if I find data (of any type), i assign it. If
834 # not, the column's empty.
835 #
836 # There are a lot of caveats in the code, so take your time to explore it.
837 #
838 # ToDo: Better complete this description
839 #
841 record = OrderedDict()
842 taggedItems = OrderedDict()
843 taggedItemsParsed = False
845 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(tag)
846 #dataDefinitionHeader.dump()
847 variableDataBytesProcessed = (dataDefinitionHeader['LastVariableDataType'] - 127) * 2
848 prevItemLen = 0
849 tagLen = len(tag)
850 fixedSizeOffset = len(dataDefinitionHeader)
851 variableSizeOffset = dataDefinitionHeader['VariableSizeOffset']
853 columns = cursor['TableData']['Columns']
855 for column in list(columns.keys()):
856 if filter_tables is not None: 856 ↛ 859line 856 didn't jump to line 859, because the condition on line 856 was never false
857 if column not in filter_tables:
858 continue
859 columnRecord = columns[column]['Record']
860 #columnRecord.dump()
861 if columnRecord['Identifier'] <= dataDefinitionHeader['LastFixedSize']: 861 ↛ 863line 861 didn't jump to line 863, because the condition on line 861 was never true
862 # Fixed Size column data type, still available data
863 record[column] = tag[fixedSizeOffset:][:columnRecord['SpaceUsage']]
864 fixedSizeOffset += columnRecord['SpaceUsage']
866 elif 127 < columnRecord['Identifier'] <= dataDefinitionHeader['LastVariableDataType']: 866 ↛ 868line 866 didn't jump to line 868, because the condition on line 866 was never true
867 # Variable data type
868 index = columnRecord['Identifier'] - 127 - 1
869 itemLen = unpack('<H',tag[variableSizeOffset+index*2:][:2])[0]
871 if itemLen & 0x8000:
872 # Empty item
873 itemLen = prevItemLen
874 record[column] = None
875 else:
876 itemValue = tag[variableSizeOffset+variableDataBytesProcessed:][:itemLen-prevItemLen]
877 record[column] = itemValue
879 #if columnRecord['Identifier'] <= dataDefinitionHeader['LastVariableDataType']:
880 variableDataBytesProcessed +=itemLen-prevItemLen
882 prevItemLen = itemLen
884 elif columnRecord['Identifier'] > 255: 884 ↛ 947line 884 didn't jump to line 947, because the condition on line 884 was never false
885 # Have we parsed the tagged items already?
886 if taggedItemsParsed is False and (variableDataBytesProcessed+variableSizeOffset) < tagLen:
887 index = variableDataBytesProcessed+variableSizeOffset
888 #hexdump(tag[index:])
889 endOfVS = self.__pageSize
890 firstOffsetTag = (unpack('<H', tag[index+2:][:2])[0] & 0x3fff) + variableDataBytesProcessed+variableSizeOffset
891 while True:
892 taggedIdentifier = unpack('<H', tag[index:][:2])[0]
893 index += 2
894 taggedOffset = (unpack('<H', tag[index:][:2])[0] & 0x3fff)
895 # As of Windows 7 and later ( version 0x620 revision 0x11) the
896 # tagged data type flags are always present
897 if self.__DBHeader['Version'] == 0x620 and self.__DBHeader['FileFormatRevision'] >= 17 and self.__DBHeader['PageSize'] > 8192: 897 ↛ 898line 897 didn't jump to line 898, because the condition on line 897 was never true
898 flagsPresent = 1
899 else:
900 flagsPresent = (unpack('<H', tag[index:][:2])[0] & 0x4000)
901 index += 2
902 if taggedOffset < endOfVS:
903 endOfVS = taggedOffset
904 taggedItems[taggedIdentifier] = (taggedOffset, tagLen, flagsPresent)
905 #print "ID: %d, Offset:%d, firstOffset:%d, index:%d, flag: 0x%x" % (taggedIdentifier, taggedOffset,firstOffsetTag,index, flagsPresent)
906 if index >= firstOffsetTag:
907 # We reached the end of the variable size array
908 break
910 # Calculate length of variable items
911 # Ugly.. should be redone
912 prevKey = list(taggedItems.keys())[0]
913 for i in range(1,len(taggedItems)):
914 offset0, length, flags = taggedItems[prevKey]
915 offset, _, _ = list(taggedItems.items())[i][1]
916 taggedItems[prevKey] = (offset0, offset-offset0, flags)
917 #print "ID: %d, Offset: %d, Len: %d, flags: %d" % (prevKey, offset0, offset-offset0, flags)
918 prevKey = list(taggedItems.keys())[i]
919 taggedItemsParsed = True
921 # Tagged data type
922 if columnRecord['Identifier'] in taggedItems:
923 offsetItem = variableDataBytesProcessed + variableSizeOffset + taggedItems[columnRecord['Identifier']][0]
924 itemSize = taggedItems[columnRecord['Identifier']][1]
925 # If item have flags, we should skip them
926 if taggedItems[columnRecord['Identifier']][2] > 0:
927 itemFlag = ord(tag[offsetItem:offsetItem+1])
928 offsetItem += 1
929 itemSize -= 1
930 else:
931 itemFlag = 0
933 #print "ID: %d, itemFlag: 0x%x" %( columnRecord['Identifier'], itemFlag)
934 if itemFlag & (TAGGED_DATA_TYPE_COMPRESSED ): 934 ↛ 935line 934 didn't jump to line 935, because the condition on line 934 was never true
935 LOG.error('Unsupported tag column: %s, flag:0x%x' % (column, itemFlag))
936 record[column] = None
937 elif itemFlag & TAGGED_DATA_TYPE_MULTI_VALUE: 937 ↛ 939line 937 didn't jump to line 939, because the condition on line 937 was never true
938 # ToDo: Parse multi-values properly
939 LOG.debug('Multivalue detected in column %s, returning raw results' % (column))
940 record[column] = (hexlify(tag[offsetItem:][:itemSize]),)
941 else:
942 record[column] = tag[offsetItem:][:itemSize]
944 else:
945 record[column] = None
946 else:
947 record[column] = None
949 # If we understand the data type, we unpack it and cast it accordingly
950 # otherwise, we just encode it in hex
951 if type(record[column]) is tuple: 951 ↛ 953line 951 didn't jump to line 953, because the condition on line 951 was never true
952 # A multi value data, we won't decode it, just leave it this way
953 record[column] = record[column][0]
954 elif columnRecord['ColumnType'] == JET_coltypText or columnRecord['ColumnType'] == JET_coltypLongText:
955 # Let's handle strings
956 if record[column] is not None:
957 if columnRecord['CodePage'] not in StringCodePages: 957 ↛ 958line 957 didn't jump to line 958, because the condition on line 957 was never true
958 raise Exception('Unknown codepage 0x%x'% columnRecord['CodePage'])
959 stringDecoder = StringCodePages[columnRecord['CodePage']]
961 try:
962 record[column] = record[column].decode(stringDecoder)
963 except Exception:
964 LOG.debug("Exception:", exc_info=True)
965 LOG.debug('Fixing Record[%r][%d]: %r' % (column, columnRecord['ColumnType'], record[column]))
966 record[column] = record[column].decode(stringDecoder, "replace")
967 pass
968 else:
969 unpackData = ColumnTypeSize[columnRecord['ColumnType']]
970 if record[column] is not None:
971 if unpackData is None:
972 record[column] = hexlify(record[column])
973 else:
974 unpackStr = unpackData[1]
975 record[column] = unpack(unpackStr, record[column])[0]
977 return record