Hide keyboard shortcuts

Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1# Impacket - Collection of Python classes for working with network protocols. 

2# 

3# SECUREAUTH LABS. Copyright (C) 2018 SecureAuth Corporation. All rights reserved. 

4# 

5# This software is provided under a slightly modified version 

6# of the Apache Software License. See the accompanying LICENSE file 

7# for more information. 

8# 

9# Description: 

10# Microsoft Extensive Storage Engine parser, just focused on trying 

11# to parse NTDS.dit files (not meant as a full parser, although it might work) 

12# 

13# Author: 

14# Alberto Solino (@agsolino) 

15# 

16# Reference for: 

17# Structure 

18#  

19# Excellent reference done by Joachim Metz 

20# - http://forensic-proof.com/wp-content/uploads/2011/07/Extensible-Storage-Engine-ESE-Database-File-EDB-format.pdf 

21# 

22# ToDo:  

23# [ ] Parse multi-values properly 

24# [ ] Support long values properly 

25# 

26 

27from __future__ import division 

28from __future__ import print_function 

29from impacket import LOG 

30try: 

31 from collections import OrderedDict 

32except: 

33 try: 

34 from ordereddict.ordereddict import OrderedDict 

35 except: 

36 from ordereddict import OrderedDict 

37from impacket.structure import Structure, hexdump 

38from struct import unpack 

39from binascii import hexlify 

40from six import b 

41 

42# Constants 

43 

44FILE_TYPE_DATABASE = 0 

45FILE_TYPE_STREAMING_FILE = 1 

46 

47# Database state 

48JET_dbstateJustCreated = 1 

49JET_dbstateDirtyShutdown = 2 

50JET_dbstateCleanShutdown = 3 

51JET_dbstateBeingConverted = 4 

52JET_dbstateForceDetach = 5 

53 

54# Page Flags 

55FLAGS_ROOT = 1 

56FLAGS_LEAF = 2 

57FLAGS_PARENT = 4 

58FLAGS_EMPTY = 8 

59FLAGS_SPACE_TREE = 0x20 

60FLAGS_INDEX = 0x40 

61FLAGS_LONG_VALUE = 0x80 

62FLAGS_NEW_FORMAT = 0x2000 

63FLAGS_NEW_CHECKSUM = 0x2000 

64 

65# Tag Flags 

66TAG_UNKNOWN = 0x1 

67TAG_DEFUNCT = 0x2 

68TAG_COMMON = 0x4 

69 

70# Fixed Page Numbers 

71DATABASE_PAGE_NUMBER = 1 

72CATALOG_PAGE_NUMBER = 4 

73CATALOG_BACKUP_PAGE_NUMBER = 24 

74 

75# Fixed FatherDataPages 

76DATABASE_FDP = 1 

77CATALOG_FDP = 2 

78CATALOG_BACKUP_FDP = 3 

79 

80# Catalog Types 

81CATALOG_TYPE_TABLE = 1 

82CATALOG_TYPE_COLUMN = 2 

83CATALOG_TYPE_INDEX = 3 

84CATALOG_TYPE_LONG_VALUE = 4 

85CATALOG_TYPE_CALLBACK = 5 

86 

87# Column Types 

88JET_coltypNil = 0 

89JET_coltypBit = 1 

90JET_coltypUnsignedByte = 2 

91JET_coltypShort = 3 

92JET_coltypLong = 4 

93JET_coltypCurrency = 5 

94JET_coltypIEEESingle = 6 

95JET_coltypIEEEDouble = 7 

96JET_coltypDateTime = 8 

97JET_coltypBinary = 9 

98JET_coltypText = 10 

99JET_coltypLongBinary = 11 

100JET_coltypLongText = 12 

101JET_coltypSLV = 13 

102JET_coltypUnsignedLong = 14 

103JET_coltypLongLong = 15 

104JET_coltypGUID = 16 

105JET_coltypUnsignedShort= 17 

106JET_coltypMax = 18 

107 

108ColumnTypeToName = { 

109 JET_coltypNil : 'NULL', 

110 JET_coltypBit : 'Boolean', 

111 JET_coltypUnsignedByte : 'Signed byte', 

112 JET_coltypShort : 'Signed short', 

113 JET_coltypLong : 'Signed long', 

114 JET_coltypCurrency : 'Currency', 

115 JET_coltypIEEESingle : 'Single precision FP', 

116 JET_coltypIEEEDouble : 'Double precision FP', 

117 JET_coltypDateTime : 'DateTime', 

118 JET_coltypBinary : 'Binary', 

119 JET_coltypText : 'Text', 

120 JET_coltypLongBinary : 'Long Binary', 

121 JET_coltypLongText : 'Long Text', 

122 JET_coltypSLV : 'Obsolete', 

123 JET_coltypUnsignedLong : 'Unsigned long', 

124 JET_coltypLongLong : 'Long long', 

125 JET_coltypGUID : 'GUID', 

126 JET_coltypUnsignedShort: 'Unsigned short', 

127 JET_coltypMax : 'Max', 

128} 

129 

130ColumnTypeSize = { 

131 JET_coltypNil : None, 

132 JET_coltypBit : (1,'B'), 

133 JET_coltypUnsignedByte : (1,'B'), 

134 JET_coltypShort : (2,'<h'), 

135 JET_coltypLong : (4,'<l'), 

136 JET_coltypCurrency : (8,'<Q'), 

137 JET_coltypIEEESingle : (4,'<f'), 

138 JET_coltypIEEEDouble : (8,'<d'), 

139 JET_coltypDateTime : (8,'<Q'), 

140 JET_coltypBinary : None, 

141 JET_coltypText : None, 

142 JET_coltypLongBinary : None, 

143 JET_coltypLongText : None, 

144 JET_coltypSLV : None, 

145 JET_coltypUnsignedLong : (4,'<L'), 

146 JET_coltypLongLong : (8,'<Q'), 

147 JET_coltypGUID : (16,'16s'), 

148 JET_coltypUnsignedShort: (2,'<H'), 

149 JET_coltypMax : None, 

150} 

151 

152# Tagged Data Type Flags 

153TAGGED_DATA_TYPE_VARIABLE_SIZE = 1 

154TAGGED_DATA_TYPE_COMPRESSED = 2 

155TAGGED_DATA_TYPE_STORED = 4 

156TAGGED_DATA_TYPE_MULTI_VALUE = 8 

157TAGGED_DATA_TYPE_WHO_KNOWS = 10 

158 

159# Code pages 

160CODEPAGE_UNICODE = 1200 

161CODEPAGE_ASCII = 20127 

162CODEPAGE_WESTERN = 1252 

163 

164StringCodePages = { 

165 CODEPAGE_UNICODE : 'utf-16le', 

166 CODEPAGE_ASCII : 'ascii', 

167 CODEPAGE_WESTERN : 'cp1252', 

168} 

169 

170# Structures 

171 

172TABLE_CURSOR = { 

173 'TableData' : b'', 

174 'FatherDataPageNumber': 0, 

175 'CurrentPageData' : b'', 

176 'CurrentTag' : 0, 

177} 

178 

179class ESENT_JET_SIGNATURE(Structure): 

180 structure = ( 

181 ('Random','<L=0'), 

182 ('CreationTime','<Q=0'), 

183 ('NetBiosName','16s=b""'), 

184 ) 

185 

186class ESENT_DB_HEADER(Structure): 

187 structure = ( 

188 ('CheckSum','<L=0'), 

189 ('Signature','"\xef\xcd\xab\x89'), 

190 ('Version','<L=0'), 

191 ('FileType','<L=0'), 

192 ('DBTime','<Q=0'), 

193 ('DBSignature',':',ESENT_JET_SIGNATURE), 

194 ('DBState','<L=0'), 

195 ('ConsistentPosition','<Q=0'), 

196 ('ConsistentTime','<Q=0'), 

197 ('AttachTime','<Q=0'), 

198 ('AttachPosition','<Q=0'), 

199 ('DetachTime','<Q=0'), 

200 ('DetachPosition','<Q=0'), 

201 ('LogSignature',':',ESENT_JET_SIGNATURE), 

202 ('Unknown','<L=0'), 

203 ('PreviousBackup','24s=b""'), 

204 ('PreviousIncBackup','24s=b""'), 

205 ('CurrentFullBackup','24s=b""'), 

206 ('ShadowingDisables','<L=0'), 

207 ('LastObjectID','<L=0'), 

208 ('WindowsMajorVersion','<L=0'), 

209 ('WindowsMinorVersion','<L=0'), 

210 ('WindowsBuildNumber','<L=0'), 

211 ('WindowsServicePackNumber','<L=0'), 

212 ('FileFormatRevision','<L=0'), 

213 ('PageSize','<L=0'), 

214 ('RepairCount','<L=0'), 

215 ('RepairTime','<Q=0'), 

216 ('Unknown2','28s=b""'), 

217 ('ScrubTime','<Q=0'), 

218 ('RequiredLog','<Q=0'), 

219 ('UpgradeExchangeFormat','<L=0'), 

220 ('UpgradeFreePages','<L=0'), 

221 ('UpgradeSpaceMapPages','<L=0'), 

222 ('CurrentShadowBackup','24s=b""'), 

223 ('CreationFileFormatVersion','<L=0'), 

224 ('CreationFileFormatRevision','<L=0'), 

225 ('Unknown3','16s=b""'), 

226 ('OldRepairCount','<L=0'), 

227 ('ECCCount','<L=0'), 

228 ('LastECCTime','<Q=0'), 

229 ('OldECCFixSuccessCount','<L=0'), 

230 ('ECCFixErrorCount','<L=0'), 

231 ('LastECCFixErrorTime','<Q=0'), 

232 ('OldECCFixErrorCount','<L=0'), 

233 ('BadCheckSumErrorCount','<L=0'), 

234 ('LastBadCheckSumTime','<Q=0'), 

235 ('OldCheckSumErrorCount','<L=0'), 

236 ('CommittedLog','<L=0'), 

237 ('PreviousShadowCopy','24s=b""'), 

238 ('PreviousDifferentialBackup','24s=b""'), 

239 ('Unknown4','40s=b""'), 

240 ('NLSMajorVersion','<L=0'), 

241 ('NLSMinorVersion','<L=0'), 

242 ('Unknown5','148s=b""'), 

243 ('UnknownFlags','<L=0'), 

244 ) 

245 

246class ESENT_PAGE_HEADER(Structure): 

247 structure_2003_SP0 = ( 

248 ('CheckSum','<L=0'), 

249 ('PageNumber','<L=0'), 

250 ) 

251 structure_0x620_0x0b = ( 

252 ('CheckSum','<L=0'), 

253 ('ECCCheckSum','<L=0'), 

254 ) 

255 structure_win7 = ( 

256 ('CheckSum','<Q=0'), 

257 ) 

258 common = ( 

259 ('LastModificationTime','<Q=0'), 

260 ('PreviousPageNumber','<L=0'), 

261 ('NextPageNumber','<L=0'), 

262 ('FatherDataPage','<L=0'), 

263 ('AvailableDataSize','<H=0'), 

264 ('AvailableUncommittedDataSize','<H=0'), 

265 ('FirstAvailableDataOffset','<H=0'), 

266 ('FirstAvailablePageTag','<H=0'), 

267 ('PageFlags','<L=0'), 

268 ) 

269 extended_win7 = ( 

270 ('ExtendedCheckSum1','<Q=0'), 

271 ('ExtendedCheckSum2','<Q=0'), 

272 ('ExtendedCheckSum3','<Q=0'), 

273 ('PageNumber','<Q=0'), 

274 ('Unknown','<Q=0'), 

275 ) 

276 def __init__(self, version, revision, pageSize=8192, data=None): 

277 if (version < 0x620) or (version == 0x620 and revision < 0x0b): 277 ↛ 279line 277 didn't jump to line 279, because the condition on line 277 was never true

278 # For sure the old format 

279 self.structure = self.structure_2003_SP0 + self.common 

280 elif version == 0x620 and revision < 0x11: 280 ↛ 282line 280 didn't jump to line 282, because the condition on line 280 was never true

281 # Exchange 2003 SP1 and Windows Vista and later 

282 self.structure = self.structure_0x620_0x0b + self.common 

283 else: 

284 # Windows 7 and later 

285 self.structure = self.structure_win7 + self.common 

286 if pageSize > 8192: 286 ↛ 287line 286 didn't jump to line 287, because the condition on line 286 was never true

287 self.structure += self.extended_win7 

288 

289 Structure.__init__(self,data) 

290 

291class ESENT_ROOT_HEADER(Structure): 

292 structure = ( 

293 ('InitialNumberOfPages','<L=0'), 

294 ('ParentFatherDataPage','<L=0'), 

295 ('ExtentSpace','<L=0'), 

296 ('SpaceTreePageNumber','<L=0'), 

297 ) 

298 

299class ESENT_BRANCH_HEADER(Structure): 

300 structure = ( 

301 ('CommonPageKey',':'), 

302 ) 

303 

304class ESENT_BRANCH_ENTRY(Structure): 

305 common = ( 

306 ('CommonPageKeySize','<H=0'), 

307 ) 

308 structure = ( 

309 ('LocalPageKeySize','<H=0'), 

310 ('_LocalPageKey','_-LocalPageKey','self["LocalPageKeySize"]'), 

311 ('LocalPageKey',':'), 

312 ('ChildPageNumber','<L=0'), 

313 ) 

314 def __init__(self, flags, data=None): 

315 if flags & TAG_COMMON > 0: 

316 # Include the common header 

317 self.structure = self.common + self.structure 

318 Structure.__init__(self,data) 

319 

320class ESENT_LEAF_HEADER(Structure): 

321 structure = ( 

322 ('CommonPageKey',':'), 

323 ) 

324 

325class ESENT_LEAF_ENTRY(Structure): 

326 common = ( 

327 ('CommonPageKeySize','<H=0'), 

328 ) 

329 structure = ( 

330 ('LocalPageKeySize','<H=0'), 

331 ('_LocalPageKey','_-LocalPageKey','self["LocalPageKeySize"]'), 

332 ('LocalPageKey',':'), 

333 ('EntryData',':'), 

334 ) 

335 def __init__(self, flags, data=None): 

336 if flags & TAG_COMMON > 0: 

337 # Include the common header 

338 self.structure = self.common + self.structure 

339 Structure.__init__(self,data) 

340 

341class ESENT_SPACE_TREE_HEADER(Structure): 

342 structure = ( 

343 ('Unknown','<Q=0'), 

344 ) 

345 

346class ESENT_SPACE_TREE_ENTRY(Structure): 

347 structure = ( 

348 ('PageKeySize','<H=0'), 

349 ('LastPageNumber','<L=0'), 

350 ('NumberOfPages','<L=0'), 

351 ) 

352 

353class ESENT_INDEX_ENTRY(Structure): 

354 structure = ( 

355 ('RecordPageKey',':'), 

356 ) 

357 

358class ESENT_DATA_DEFINITION_HEADER(Structure): 

359 structure = ( 

360 ('LastFixedSize','<B=0'), 

361 ('LastVariableDataType','<B=0'), 

362 ('VariableSizeOffset','<H=0'), 

363 ) 

364 

365class ESENT_CATALOG_DATA_DEFINITION_ENTRY(Structure): 

366 fixed = ( 

367 ('FatherDataPageID','<L=0'), 

368 ('Type','<H=0'), 

369 ('Identifier','<L=0'), 

370 ) 

371 

372 column_stuff = ( 

373 ('ColumnType','<L=0'), 

374 ('SpaceUsage','<L=0'), 

375 ('ColumnFlags','<L=0'), 

376 ('CodePage','<L=0'), 

377 ) 

378 

379 other = ( 

380 ('FatherDataPageNumber','<L=0'), 

381 ) 

382 

383 table_stuff = ( 

384 ('SpaceUsage','<L=0'), 

385# ('TableFlags','<L=0'), 

386# ('InitialNumberOfPages','<L=0'), 

387 ) 

388 

389 index_stuff = ( 

390 ('SpaceUsage','<L=0'), 

391 ('IndexFlags','<L=0'), 

392 ('Locale','<L=0'), 

393 ) 

394 

395 lv_stuff = ( 

396 ('SpaceUsage','<L=0'), 

397# ('LVFlags','<L=0'), 

398# ('InitialNumberOfPages','<L=0'), 

399 ) 

400 common = ( 

401# ('RootFlag','<B=0'), 

402# ('RecordOffset','<H=0'), 

403# ('LCMapFlags','<L=0'), 

404# ('KeyMost','<H=0'), 

405 ('Trailing',':'), 

406 ) 

407 

408 def __init__(self,data): 

409 # Depending on the type of data we'll end up building a different struct 

410 dataType = unpack('<H', data[4:][:2])[0] 

411 self.structure = self.fixed 

412 

413 if dataType == CATALOG_TYPE_TABLE: 

414 self.structure += self.other + self.table_stuff 

415 elif dataType == CATALOG_TYPE_COLUMN: 

416 self.structure += self.column_stuff 

417 elif dataType == CATALOG_TYPE_INDEX: 

418 self.structure += self.other + self.index_stuff 

419 elif dataType == CATALOG_TYPE_LONG_VALUE: 419 ↛ 421line 419 didn't jump to line 421, because the condition on line 419 was never false

420 self.structure += self.other + self.lv_stuff 

421 elif dataType == CATALOG_TYPE_CALLBACK: 

422 raise Exception('CallBack types not supported!') 

423 else: 

424 LOG.error('Unknown catalog type 0x%x' % dataType) 

425 self.structure = () 

426 Structure.__init__(self,data) 

427 

428 self.structure += self.common 

429 

430 Structure.__init__(self,data) 

431 

432 

433#def pretty_print(x): 

434# if x in '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ ': 

435# return x 

436# else: 

437# return '.' 

438# 

439#def hexdump(data): 

440# x=str(data) 

441# strLen = len(x) 

442# i = 0 

443# while i < strLen: 

444# print "%04x " % i, 

445# for j in range(16): 

446# if i+j < strLen: 

447# print "%02X" % ord(x[i+j]), 

448# 

449# else: 

450# print " ", 

451# if j%16 == 7: 

452# print "", 

453# print " ", 

454# print ''.join(pretty_print(x) for x in x[i:i+16] ) 

455# i += 16 

456 

457def getUnixTime(t): 

458 t -= 116444736000000000 

459 t //= 10000000 

460 return t 

461 

462class ESENT_PAGE: 

463 def __init__(self, db, data=None): 

464 self.__DBHeader = db 

465 self.data = data 

466 self.record = None 

467 if data is not None: 467 ↛ exitline 467 didn't return from function '__init__', because the condition on line 467 was never false

468 self.record = ESENT_PAGE_HEADER(self.__DBHeader['Version'], self.__DBHeader['FileFormatRevision'], self.__DBHeader['PageSize'], data) 

469 

470 def printFlags(self): 

471 flags = self.record['PageFlags'] 

472 if flags & FLAGS_EMPTY: 

473 print("\tEmpty") 

474 if flags & FLAGS_INDEX: 

475 print("\tIndex") 

476 if flags & FLAGS_LEAF: 

477 print("\tLeaf") 

478 else: 

479 print("\tBranch") 

480 if flags & FLAGS_LONG_VALUE: 

481 print("\tLong Value") 

482 if flags & FLAGS_NEW_CHECKSUM: 

483 print("\tNew Checksum") 

484 if flags & FLAGS_NEW_FORMAT: 

485 print("\tNew Format") 

486 if flags & FLAGS_PARENT: 

487 print("\tParent") 

488 if flags & FLAGS_ROOT: 

489 print("\tRoot") 

490 if flags & FLAGS_SPACE_TREE: 

491 print("\tSpace Tree") 

492 

493 def dump(self): 

494 baseOffset = len(self.record) 

495 self.record.dump() 

496 tags = self.data[-4*self.record['FirstAvailablePageTag']:] 

497 

498 print("FLAGS: ") 

499 self.printFlags() 

500 

501 print() 

502 

503 for i in range(self.record['FirstAvailablePageTag']): 

504 tag = tags[-4:] 

505 if self.__DBHeader['Version'] == 0x620 and self.__DBHeader['FileFormatRevision'] > 11 and self.__DBHeader['PageSize'] > 8192: 

506 valueSize = unpack('<H', tag[:2])[0] & 0x7fff 

507 valueOffset = unpack('<H',tag[2:])[0] & 0x7fff 

508 hexdump((self.data[baseOffset+valueOffset:][:6])) 

509 pageFlags = ord(self.data[baseOffset+valueOffset:][1]) >> 5 

510 #print "TAG FLAG: 0x%x " % (unpack('<L', self.data[baseOffset+valueOffset:][:4]) ) >> 5 

511 #print "TAG FLAG: 0x " , ord(self.data[baseOffset+valueOffset:][0]) 

512 else: 

513 valueSize = unpack('<H', tag[:2])[0] & 0x1fff 

514 pageFlags = (unpack('<H', tag[2:])[0] & 0xe000) >> 13 

515 valueOffset = unpack('<H',tag[2:])[0] & 0x1fff 

516 

517 print("TAG %-8d offset:0x%-6x flags:0x%-4x valueSize:0x%x" % (i,valueOffset,pageFlags,valueSize)) 

518 #hexdump(self.getTag(i)[1]) 

519 tags = tags[:-4] 

520 

521 if self.record['PageFlags'] & FLAGS_ROOT > 0: 

522 rootHeader = ESENT_ROOT_HEADER(self.getTag(0)[1]) 

523 rootHeader.dump() 

524 elif self.record['PageFlags'] & FLAGS_LEAF == 0: 

525 # Branch Header 

526 flags, data = self.getTag(0) 

527 branchHeader = ESENT_BRANCH_HEADER(data) 

528 branchHeader.dump() 

529 else: 

530 # Leaf Header 

531 flags, data = self.getTag(0) 

532 if self.record['PageFlags'] & FLAGS_SPACE_TREE > 0: 

533 # Space Tree 

534 spaceTreeHeader = ESENT_SPACE_TREE_HEADER(data) 

535 spaceTreeHeader.dump() 

536 else: 

537 leafHeader = ESENT_LEAF_HEADER(data) 

538 leafHeader.dump() 

539 

540 # Print the leaf/branch tags 

541 for tagNum in range(1,self.record['FirstAvailablePageTag']): 

542 flags, data = self.getTag(tagNum) 

543 if self.record['PageFlags'] & FLAGS_LEAF == 0: 

544 # Branch page 

545 branchEntry = ESENT_BRANCH_ENTRY(flags, data) 

546 branchEntry.dump() 

547 elif self.record['PageFlags'] & FLAGS_LEAF > 0: 

548 # Leaf page 

549 if self.record['PageFlags'] & FLAGS_SPACE_TREE > 0: 

550 # Space Tree 

551 spaceTreeEntry = ESENT_SPACE_TREE_ENTRY(data) 

552 #spaceTreeEntry.dump() 

553 

554 elif self.record['PageFlags'] & FLAGS_INDEX > 0: 

555 # Index Entry 

556 indexEntry = ESENT_INDEX_ENTRY(data) 

557 #indexEntry.dump() 

558 elif self.record['PageFlags'] & FLAGS_LONG_VALUE > 0: 

559 # Long Page Value 

560 raise Exception('Long value still not supported') 

561 else: 

562 # Table Value 

563 leafEntry = ESENT_LEAF_ENTRY(flags, data) 

564 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(leafEntry['EntryData']) 

565 dataDefinitionHeader.dump() 

566 catalogEntry = ESENT_CATALOG_DATA_DEFINITION_ENTRY(leafEntry['EntryData'][len(dataDefinitionHeader):]) 

567 catalogEntry.dump() 

568 hexdump(leafEntry['EntryData']) 

569 

570 def getTag(self, tagNum): 

571 if self.record['FirstAvailablePageTag'] < tagNum: 571 ↛ 572line 571 didn't jump to line 572, because the condition on line 571 was never true

572 raise Exception('Trying to grab an unknown tag 0x%x' % tagNum) 

573 

574 tags = self.data[-4*self.record['FirstAvailablePageTag']:] 

575 baseOffset = len(self.record) 

576 for i in range(tagNum): 

577 tags = tags[:-4] 

578 

579 tag = tags[-4:] 

580 

581 if self.__DBHeader['Version'] == 0x620 and self.__DBHeader['FileFormatRevision'] >= 17 and self.__DBHeader['PageSize'] > 8192: 581 ↛ 582line 581 didn't jump to line 582, because the condition on line 581 was never true

582 valueSize = unpack('<H', tag[:2])[0] & 0x7fff 

583 valueOffset = unpack('<H',tag[2:])[0] & 0x7fff 

584 tmpData = list(self.data[baseOffset+valueOffset:][:valueSize]) 

585 pageFlags = ord(tmpData[1]) >> 5 

586 tmpData[1] = chr(ord(tmpData[1:2]) & 0x1f) 

587 tagData = "".join(tmpData) 

588 else: 

589 valueSize = unpack('<H', tag[:2])[0] & 0x1fff 

590 pageFlags = (unpack('<H', tag[2:])[0] & 0xe000) >> 13 

591 valueOffset = unpack('<H',tag[2:])[0] & 0x1fff 

592 tagData = self.data[baseOffset+valueOffset:][:valueSize] 

593 

594 #return pageFlags, self.data[baseOffset+valueOffset:][:valueSize] 

595 return pageFlags, tagData 

596 

597class ESENT_DB: 

598 def __init__(self, fileName, pageSize = 8192, isRemote = False): 

599 self.__fileName = fileName 

600 self.__pageSize = pageSize 

601 self.__DB = None 

602 self.__DBHeader = None 

603 self.__totalPages = None 

604 self.__tables = OrderedDict() 

605 self.__currentTable = None 

606 self.__isRemote = isRemote 

607 self.mountDB() 

608 

609 def mountDB(self): 

610 LOG.debug("Mounting DB...") 

611 if self.__isRemote is True: 611 ↛ 615line 611 didn't jump to line 615, because the condition on line 611 was never false

612 self.__DB = self.__fileName 

613 self.__DB.open() 

614 else: 

615 self.__DB = open(self.__fileName,"rb") 

616 mainHeader = self.getPage(-1) 

617 self.__DBHeader = ESENT_DB_HEADER(mainHeader) 

618 self.__pageSize = self.__DBHeader['PageSize'] 

619 self.__DB.seek(0,2) 

620 self.__totalPages = (self.__DB.tell() // self.__pageSize) -2 

621 LOG.debug("Database Version:0x%x, Revision:0x%x"% (self.__DBHeader['Version'], self.__DBHeader['FileFormatRevision'])) 

622 LOG.debug("Page Size: %d" % self.__pageSize) 

623 LOG.debug("Total Pages in file: %d" % self.__totalPages) 

624 self.parseCatalog(CATALOG_PAGE_NUMBER) 

625 

626 def printCatalog(self): 

627 indent = ' ' 

628 

629 print("Database version: 0x%x, 0x%x" % (self.__DBHeader['Version'], self.__DBHeader['FileFormatRevision'] )) 

630 print("Page size: %d " % self.__pageSize) 

631 print("Number of pages: %d" % self.__totalPages) 

632 print() 

633 print("Catalog for %s" % self.__fileName) 

634 for table in list(self.__tables.keys()): 

635 print("[%s]" % table.decode('utf8')) 

636 print("%sColumns " % indent) 

637 for column in list(self.__tables[table]['Columns'].keys()): 

638 record = self.__tables[table]['Columns'][column]['Record'] 

639 print("%s%-5d%-30s%s" % (indent*2, record['Identifier'], column.decode('utf-8'),ColumnTypeToName[record['ColumnType']])) 

640 print("%sIndexes"% indent) 

641 for index in list(self.__tables[table]['Indexes'].keys()): 

642 print("%s%s" % (indent*2, index.decode('utf-8'))) 

643 print("") 

644 

645 def __addItem(self, entry): 

646 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData']) 

647 catalogEntry = ESENT_CATALOG_DATA_DEFINITION_ENTRY(entry['EntryData'][len(dataDefinitionHeader):]) 

648 itemName = self.__parseItemName(entry) 

649 

650 if catalogEntry['Type'] == CATALOG_TYPE_TABLE: 

651 self.__tables[itemName] = OrderedDict() 

652 self.__tables[itemName]['TableEntry'] = entry 

653 self.__tables[itemName]['Columns'] = OrderedDict() 

654 self.__tables[itemName]['Indexes'] = OrderedDict() 

655 self.__tables[itemName]['LongValues'] = OrderedDict() 

656 self.__currentTable = itemName 

657 elif catalogEntry['Type'] == CATALOG_TYPE_COLUMN: 

658 self.__tables[self.__currentTable]['Columns'][itemName] = entry 

659 self.__tables[self.__currentTable]['Columns'][itemName]['Header'] = dataDefinitionHeader 

660 self.__tables[self.__currentTable]['Columns'][itemName]['Record'] = catalogEntry 

661 elif catalogEntry['Type'] == CATALOG_TYPE_INDEX: 

662 self.__tables[self.__currentTable]['Indexes'][itemName] = entry 

663 elif catalogEntry['Type'] == CATALOG_TYPE_LONG_VALUE: 663 ↛ 666line 663 didn't jump to line 666, because the condition on line 663 was never false

664 self.__addLongValue(entry) 

665 else: 

666 raise Exception('Unknown type 0x%x' % catalogEntry['Type']) 

667 

668 def __parseItemName(self,entry): 

669 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData']) 

670 

671 if dataDefinitionHeader['LastVariableDataType'] > 127: 671 ↛ 674line 671 didn't jump to line 674, because the condition on line 671 was never false

672 numEntries = dataDefinitionHeader['LastVariableDataType'] - 127 

673 else: 

674 numEntries = dataDefinitionHeader['LastVariableDataType'] 

675 

676 itemLen = unpack('<H',entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][:2])[0] 

677 itemName = entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][2*numEntries:][:itemLen] 

678 return itemName 

679 

680 def __addLongValue(self, entry): 

681 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData']) 

682 lvLen = unpack('<H',entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][:2])[0] 

683 lvName = entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][7:][:lvLen] 

684 self.__tables[self.__currentTable]['LongValues'][lvName] = entry 

685 

686 def parsePage(self, page): 

687 # Print the leaf/branch tags 

688 for tagNum in range(1,page.record['FirstAvailablePageTag']): 

689 flags, data = page.getTag(tagNum) 

690 if page.record['PageFlags'] & FLAGS_LEAF > 0: 

691 # Leaf page 

692 if page.record['PageFlags'] & FLAGS_SPACE_TREE > 0: 692 ↛ 693line 692 didn't jump to line 693, because the condition on line 692 was never true

693 pass 

694 elif page.record['PageFlags'] & FLAGS_INDEX > 0: 694 ↛ 695line 694 didn't jump to line 695, because the condition on line 694 was never true

695 pass 

696 elif page.record['PageFlags'] & FLAGS_LONG_VALUE > 0: 696 ↛ 697line 696 didn't jump to line 697, because the condition on line 696 was never true

697 pass 

698 else: 

699 # Table Value 

700 leafEntry = ESENT_LEAF_ENTRY(flags, data) 

701 self.__addItem(leafEntry) 

702 

703 def parseCatalog(self, pageNum): 

704 # Parse all the pages starting at pageNum and commit table data 

705 page = self.getPage(pageNum) 

706 self.parsePage(page) 

707 

708 for i in range(1, page.record['FirstAvailablePageTag']): 

709 flags, data = page.getTag(i) 

710 if page.record['PageFlags'] & FLAGS_LEAF == 0: 

711 # Branch page 

712 branchEntry = ESENT_BRANCH_ENTRY(flags, data) 

713 self.parseCatalog(branchEntry['ChildPageNumber']) 

714 

715 

716 def readHeader(self): 

717 LOG.debug("Reading Boot Sector for %s" % self.__volumeName) 

718 

719 def getPage(self, pageNum): 

720 LOG.debug("Trying to fetch page %d (0x%x)" % (pageNum, (pageNum+1)*self.__pageSize)) 

721 self.__DB.seek((pageNum+1)*self.__pageSize, 0) 

722 data = self.__DB.read(self.__pageSize) 

723 while len(data) < self.__pageSize: 723 ↛ 724line 723 didn't jump to line 724, because the condition on line 723 was never true

724 remaining = self.__pageSize - len(data) 

725 data += self.__DB.read(remaining) 

726 # Special case for the first page 

727 if pageNum <= 0: 

728 return data 

729 else: 

730 return ESENT_PAGE(self.__DBHeader, data) 

731 

732 def close(self): 

733 self.__DB.close() 

734 

735 def openTable(self, tableName): 

736 # Returns a cursos for later use 

737 

738 if isinstance(tableName, bytes) is not True: 738 ↛ 741line 738 didn't jump to line 741, because the condition on line 738 was never false

739 tableName = b(tableName) 

740 

741 if tableName in self.__tables: 741 ↛ 772line 741 didn't jump to line 772, because the condition on line 741 was never false

742 entry = self.__tables[tableName]['TableEntry'] 

743 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData']) 

744 catalogEntry = ESENT_CATALOG_DATA_DEFINITION_ENTRY(entry['EntryData'][len(dataDefinitionHeader):]) 

745 

746 # Let's position the cursor at the leaf levels for fast reading 

747 pageNum = catalogEntry['FatherDataPageNumber'] 

748 done = False 

749 while done is False: 

750 page = self.getPage(pageNum) 

751 if page.record['FirstAvailablePageTag'] <= 1: 751 ↛ 753line 751 didn't jump to line 753, because the condition on line 751 was never true

752 # There are no records 

753 done = True 

754 for i in range(1, page.record['FirstAvailablePageTag']): 754 ↛ 749line 754 didn't jump to line 749, because the loop on line 754 didn't complete

755 flags, data = page.getTag(i) 

756 if page.record['PageFlags'] & FLAGS_LEAF == 0: 

757 # Branch page, move on to the next page 

758 branchEntry = ESENT_BRANCH_ENTRY(flags, data) 

759 pageNum = branchEntry['ChildPageNumber'] 

760 break 

761 else: 

762 done = True 

763 break 

764 

765 cursor = TABLE_CURSOR 

766 cursor['TableData'] = self.__tables[tableName] 

767 cursor['FatherDataPageNumber'] = catalogEntry['FatherDataPageNumber'] 

768 cursor['CurrentPageData'] = page 

769 cursor['CurrentTag'] = 0 

770 return cursor 

771 else: 

772 return None 

773 

774 def __getNextTag(self, cursor): 

775 page = cursor['CurrentPageData'] 

776 

777 if cursor['CurrentTag'] >= page.record['FirstAvailablePageTag']: 

778 # No more data in this page, chau 

779 return None 

780 

781 flags, data = page.getTag(cursor['CurrentTag']) 

782 if page.record['PageFlags'] & FLAGS_LEAF > 0: 782 ↛ 795line 782 didn't jump to line 795, because the condition on line 782 was never false

783 # Leaf page 

784 if page.record['PageFlags'] & FLAGS_SPACE_TREE > 0: 784 ↛ 785line 784 didn't jump to line 785, because the condition on line 784 was never true

785 raise Exception('FLAGS_SPACE_TREE > 0') 

786 elif page.record['PageFlags'] & FLAGS_INDEX > 0: 786 ↛ 787line 786 didn't jump to line 787, because the condition on line 786 was never true

787 raise Exception('FLAGS_INDEX > 0') 

788 elif page.record['PageFlags'] & FLAGS_LONG_VALUE > 0: 788 ↛ 789line 788 didn't jump to line 789, because the condition on line 788 was never true

789 raise Exception('FLAGS_LONG_VALUE > 0') 

790 else: 

791 # Table Value 

792 leafEntry = ESENT_LEAF_ENTRY(flags, data) 

793 return leafEntry 

794 

795 return None 

796 

797 def getNextRow(self, cursor, filter_tables = None): 

798 cursor['CurrentTag'] += 1 

799 

800 tag = self.__getNextTag(cursor) 

801 #hexdump(tag) 

802 

803 if tag is None: 

804 # No more tags in this page, search for the next one on the right 

805 page = cursor['CurrentPageData'] 

806 if page.record['NextPageNumber'] == 0: 

807 # No more pages, chau 

808 return None 

809 else: 

810 cursor['CurrentPageData'] = self.getPage(page.record['NextPageNumber']) 

811 cursor['CurrentTag'] = 0 

812 return self.getNextRow(cursor, filter_tables = filter_tables) 

813 else: 

814 return self.__tagToRecord(cursor, tag['EntryData'], filter_tables = filter_tables) 

815 

816 def __tagToRecord(self, cursor, tag, filter_tables = None): 

817 # So my brain doesn't forget, the data record is composed of: 

818 # Header 

819 # Fixed Size Data (ID < 127) 

820 # The easiest to parse. Their size is fixed in the record. You can get its size 

821 # from the Column Record, field SpaceUsage 

822 # Variable Size Data (127 < ID < 255) 

823 # At VariableSizeOffset you get an array of two bytes per variable entry, pointing 

824 # to the length of the value. Values start at: 

825 # numEntries = LastVariableDataType - 127 

826 # VariableSizeOffset + numEntries * 2 (bytes) 

827 # Tagged Data ( > 255 ) 

828 # After the Variable Size Value, there's more data for the tagged values. 

829 # Right at the beginning there's another array (taggedItems), pointing to the 

830 # values, size. 

831 # 

832 # The interesting thing about this DB records is there's no need for all the columns to be there, hence 

833 # saving space. That's why I got over all the columns, and if I find data (of any type), i assign it. If  

834 # not, the column's empty. 

835 # 

836 # There are a lot of caveats in the code, so take your time to explore it.  

837 # 

838 # ToDo: Better complete this description 

839 # 

840 

841 record = OrderedDict() 

842 taggedItems = OrderedDict() 

843 taggedItemsParsed = False 

844 

845 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(tag) 

846 #dataDefinitionHeader.dump() 

847 variableDataBytesProcessed = (dataDefinitionHeader['LastVariableDataType'] - 127) * 2 

848 prevItemLen = 0 

849 tagLen = len(tag) 

850 fixedSizeOffset = len(dataDefinitionHeader) 

851 variableSizeOffset = dataDefinitionHeader['VariableSizeOffset'] 

852 

853 columns = cursor['TableData']['Columns'] 

854 

855 for column in list(columns.keys()): 

856 if filter_tables is not None: 856 ↛ 859line 856 didn't jump to line 859, because the condition on line 856 was never false

857 if column not in filter_tables: 

858 continue 

859 columnRecord = columns[column]['Record'] 

860 #columnRecord.dump() 

861 if columnRecord['Identifier'] <= dataDefinitionHeader['LastFixedSize']: 861 ↛ 863line 861 didn't jump to line 863, because the condition on line 861 was never true

862 # Fixed Size column data type, still available data 

863 record[column] = tag[fixedSizeOffset:][:columnRecord['SpaceUsage']] 

864 fixedSizeOffset += columnRecord['SpaceUsage'] 

865 

866 elif 127 < columnRecord['Identifier'] <= dataDefinitionHeader['LastVariableDataType']: 866 ↛ 868line 866 didn't jump to line 868, because the condition on line 866 was never true

867 # Variable data type 

868 index = columnRecord['Identifier'] - 127 - 1 

869 itemLen = unpack('<H',tag[variableSizeOffset+index*2:][:2])[0] 

870 

871 if itemLen & 0x8000: 

872 # Empty item 

873 itemLen = prevItemLen 

874 record[column] = None 

875 else: 

876 itemValue = tag[variableSizeOffset+variableDataBytesProcessed:][:itemLen-prevItemLen] 

877 record[column] = itemValue 

878 

879 #if columnRecord['Identifier'] <= dataDefinitionHeader['LastVariableDataType']: 

880 variableDataBytesProcessed +=itemLen-prevItemLen 

881 

882 prevItemLen = itemLen 

883 

884 elif columnRecord['Identifier'] > 255: 884 ↛ 947line 884 didn't jump to line 947, because the condition on line 884 was never false

885 # Have we parsed the tagged items already? 

886 if taggedItemsParsed is False and (variableDataBytesProcessed+variableSizeOffset) < tagLen: 

887 index = variableDataBytesProcessed+variableSizeOffset 

888 #hexdump(tag[index:]) 

889 endOfVS = self.__pageSize 

890 firstOffsetTag = (unpack('<H', tag[index+2:][:2])[0] & 0x3fff) + variableDataBytesProcessed+variableSizeOffset 

891 while True: 

892 taggedIdentifier = unpack('<H', tag[index:][:2])[0] 

893 index += 2 

894 taggedOffset = (unpack('<H', tag[index:][:2])[0] & 0x3fff) 

895 # As of Windows 7 and later ( version 0x620 revision 0x11) the  

896 # tagged data type flags are always present 

897 if self.__DBHeader['Version'] == 0x620 and self.__DBHeader['FileFormatRevision'] >= 17 and self.__DBHeader['PageSize'] > 8192: 897 ↛ 898line 897 didn't jump to line 898, because the condition on line 897 was never true

898 flagsPresent = 1 

899 else: 

900 flagsPresent = (unpack('<H', tag[index:][:2])[0] & 0x4000) 

901 index += 2 

902 if taggedOffset < endOfVS: 

903 endOfVS = taggedOffset 

904 taggedItems[taggedIdentifier] = (taggedOffset, tagLen, flagsPresent) 

905 #print "ID: %d, Offset:%d, firstOffset:%d, index:%d, flag: 0x%x" % (taggedIdentifier, taggedOffset,firstOffsetTag,index, flagsPresent) 

906 if index >= firstOffsetTag: 

907 # We reached the end of the variable size array 

908 break 

909 

910 # Calculate length of variable items 

911 # Ugly.. should be redone 

912 prevKey = list(taggedItems.keys())[0] 

913 for i in range(1,len(taggedItems)): 

914 offset0, length, flags = taggedItems[prevKey] 

915 offset, _, _ = list(taggedItems.items())[i][1] 

916 taggedItems[prevKey] = (offset0, offset-offset0, flags) 

917 #print "ID: %d, Offset: %d, Len: %d, flags: %d" % (prevKey, offset0, offset-offset0, flags) 

918 prevKey = list(taggedItems.keys())[i] 

919 taggedItemsParsed = True 

920 

921 # Tagged data type 

922 if columnRecord['Identifier'] in taggedItems: 

923 offsetItem = variableDataBytesProcessed + variableSizeOffset + taggedItems[columnRecord['Identifier']][0] 

924 itemSize = taggedItems[columnRecord['Identifier']][1] 

925 # If item have flags, we should skip them 

926 if taggedItems[columnRecord['Identifier']][2] > 0: 

927 itemFlag = ord(tag[offsetItem:offsetItem+1]) 

928 offsetItem += 1 

929 itemSize -= 1 

930 else: 

931 itemFlag = 0 

932 

933 #print "ID: %d, itemFlag: 0x%x" %( columnRecord['Identifier'], itemFlag) 

934 if itemFlag & (TAGGED_DATA_TYPE_COMPRESSED ): 934 ↛ 935line 934 didn't jump to line 935, because the condition on line 934 was never true

935 LOG.error('Unsupported tag column: %s, flag:0x%x' % (column, itemFlag)) 

936 record[column] = None 

937 elif itemFlag & TAGGED_DATA_TYPE_MULTI_VALUE: 937 ↛ 939line 937 didn't jump to line 939, because the condition on line 937 was never true

938 # ToDo: Parse multi-values properly 

939 LOG.debug('Multivalue detected in column %s, returning raw results' % (column)) 

940 record[column] = (hexlify(tag[offsetItem:][:itemSize]),) 

941 else: 

942 record[column] = tag[offsetItem:][:itemSize] 

943 

944 else: 

945 record[column] = None 

946 else: 

947 record[column] = None 

948 

949 # If we understand the data type, we unpack it and cast it accordingly 

950 # otherwise, we just encode it in hex 

951 if type(record[column]) is tuple: 951 ↛ 953line 951 didn't jump to line 953, because the condition on line 951 was never true

952 # A multi value data, we won't decode it, just leave it this way 

953 record[column] = record[column][0] 

954 elif columnRecord['ColumnType'] == JET_coltypText or columnRecord['ColumnType'] == JET_coltypLongText: 

955 # Let's handle strings 

956 if record[column] is not None: 

957 if columnRecord['CodePage'] not in StringCodePages: 957 ↛ 958line 957 didn't jump to line 958, because the condition on line 957 was never true

958 raise Exception('Unknown codepage 0x%x'% columnRecord['CodePage']) 

959 stringDecoder = StringCodePages[columnRecord['CodePage']] 

960 

961 try: 

962 record[column] = record[column].decode(stringDecoder) 

963 except Exception: 

964 LOG.debug("Exception:", exc_info=True) 

965 LOG.debug('Fixing Record[%r][%d]: %r' % (column, columnRecord['ColumnType'], record[column])) 

966 record[column] = record[column].decode(stringDecoder, "replace") 

967 pass 

968 else: 

969 unpackData = ColumnTypeSize[columnRecord['ColumnType']] 

970 if record[column] is not None: 

971 if unpackData is None: 

972 record[column] = hexlify(record[column]) 

973 else: 

974 unpackStr = unpackData[1] 

975 record[column] = unpack(unpackStr, record[column])[0] 

976 

977 return record