# optimizations in python

This topic is 4150 days old which is more than the 365 day threshold we allow for new replies. Please post a new topic.

## Recommended Posts

Currently I'm in the porcess of writing a program in python that opens and edits a modfile for a certain game. These files can be anywhere from 1mb to 300mb. However, my program isn't that fast, it takes around 5 minutes to load something 10mb in size. Before I elaborate further, here's my sourcecode: GUI.py
import wx
import wx.aui
import os
import cStringIO
import test
import Sorter

#-------------------
def GetMondrianData():
return '\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00 \x00\x00\x00 \x08\x06\x00\x00\x00szz\xf4\x00\x00\x00\x04sBIT\x08\x08\x08\x08|\x08d\x88\x00\x00\x00qIDATX\x85\xed\xd6;\n\x800\x10E\xd1{\xc5\x8d\xb9r\x97\x16\x0b\xad$\x8a\x82:\x16o\xda\x84pB2\x1f\x81Fa\x8c\x9c\x08\x04Z{\xcf\xa72\xbcv\xfa\xc5\x08 \x80r\x80\xfc\xa2\x0e\x1c\xe4\xba\xfaX\x1d\xd0\xde]S\x07\x02\xd8&gt;\xe1wa-\x9fQ\xe9\x86\x01\x04\x10\x00\\(Dk\x1b-\x04\xdc\x1d\x07\x14\x98;\x0bS\x7f\x7f\xf9\x13\x04\x10@\xf9X\xbe\x00\xc9 \x14K\xc1&lt;={\x00\x00\x00\x00IEND\xaeB\x82' def GetMondrianBitmap(): return wx.BitmapFromImage(GetMondrianImage()) def GetMondrianImage(): import cStringIO stream = cStringIO.StringIO(GetMondrianData()) return wx.ImageFromStream(stream) def GetMondrianIcon(): icon = wx.EmptyIcon() icon.CopyFromBitmap(GetMondrianBitmap()) return icon #-------------------------------- class ParentFrame(wx.aui.AuiMDIParentFrame): def __init__(self, parent): wx.aui.AuiMDIParentFrame.__init__(self, parent, -1, title="Tes4Editor", size=(640,480), style=wx.DEFAULT_FRAME_style) self.SetIcon(GetMondrianIcon()) mb = self.MakeMenuBar() self.SetMenuBar(mb) self.CreateStatusBar() def MakeMenuBar(self): mb = wx.MenuBar() menu = wx.Menu() item = menu.Append(-1, "Open...\tCtrl-O") self.Bind(wx.EVT_MENU, self.OnOpen, item) item = menu.Append(-1, "&Quit\tCtrl-Q") self.Bind(wx.EVT_MENU, self.OnDoClose, item) menu2 = wx.Menu() item = menu2.Append(-1, "About\tCtrl-A") self.Bind(wx.EVT_MENU, self.OnAbout, item) mb.Append(menu, "&File") mb.Append(menu2, "&Help") return mb def OnOpen(self, e): """ Open a file""" self.dirname = '' dlg = wx.FileDialog(self, "Choose a file", self.dirname, "", "esp/esm files | *esp;*.esm", wx.OPEN) if dlg.ShowModal() == wx.ID_OK: self.filename = dlg.GetFilename() self.dirname = dlg.GetDirectory() filename = os.path.join(self.dirname,self.filename) child = ChildFrame(self, self.filename, self.dirname, filename) child.Show() dlg.Destroy() def OnAbout(self, e): msg = "TES4++ - Utitlity for low level \nediting of esp/esm files." d = wx.MessageDialog(self, msg, "TES4++", wx.OK | wx.ICON_INFORMATION) d.ShowModal() d.Destroy() def OnDoClose(self, evt): self.Close() class ChildFrame(wx.aui.AuiMDIChildFrame): def __init__(self, parent, filename="test.esp", dirname="C:/Program Files/Bethesda Softworks/Oblivion/Data", name="C:/Program Files/Bethesda Softworks/Oblivion/Data/test.esp"): wx.aui.AuiMDIChildFrame.__init__(self, parent, -1, title = filename) self.filename = filename self.dirname = dirname self.name = name mb = parent.MakeMenuBar() menu = wx.Menu() item = menu.Append(-1, "&Save\tCtrl-S") self.Bind(wx.EVT_MENU, self.OnSave, item) item = menu.Append(-1, "&Save As...") self.Bind(wx.EVT_MENU, self.OnSaveAs, item) mb.Append(menu, self.filename) self.SetMenuBar(mb) self.FileObj = test.File(self.name) self.FileObj.parseRecord() for node in self.FileObj.GRUPS: node.scan() self.TreeList = [] self.Changed = False self.split = wx.SplitterWindow(self, -1) #------Tree here---------------------------------- il = wx.ImageList(16,16) self.fldridx = il.Add( wx.ArtProvider.GetBitmap(wx.ART_FOLDER, wx.ART_OTHER, (16,16))) self.fldropenidx = il.Add( wx.ArtProvider.GetBitmap(wx.ART_FILE_OPEN, wx.ART_OTHER, (16,16))) self.fileidx = il.Add( wx.ArtProvider.GetBitmap(wx.ART_NORMAL_FILE, wx.ART_OTHER, (16,16))) self.tree = wx.TreeCtrl(self.split, 1, wx.DefaultPosition, (-1,-1), wx.TR_HAS_BUTTONS) self.root = self.tree.AddRoot(self.filename) self.tree.AssignImageList(il) self.tree.SetItemImage(self.root, self.fldridx, wx.TreeItemIcon_Normal) self.tree.SetItemImage(self.root, self.fldropenidx, wx.TreeItemIcon_Expanded) self.tree.SetItemPyData(self.root, self.FileObj.GRUPS) self.tree.SetItemHasChildren(self.root, True) #Bind some tree events #self.Bind(wx.EVT_TREE_SEL_CHANGED, self.OnSelChanged, self.tree) self.Bind(wx.EVT_TREE_ITEM_EXPANDING, self.OnItemExpanding, self.tree) self.Bind(wx.EVT_TREE_ITEM_COLLAPSED, self.OnItemCollapsed, self.tree) #----------------------------------------------------- self.panel = wx.Panel(self.split, -1) self.split.SplitVertically(self.tree, self.panel, 200) sizer = wx.BoxSizer() sizer.Add(self.split, 1, wx.EXPAND) self.SetSizer(sizer) wx.CallAfter(self.Layout) (shortname, ext) = os.path.splitext(self.filename) self.popupmenu = wx.Menu() item = self.popupmenu.Append(-1, "Save") self.Bind(wx.EVT_MENU, self.OnSave, item) item = self.popupmenu.Append(-1, "Save As...") self.Bind(wx.EVT_MENU, self.OnSaveAs, item) if ext == ".esm": item = self.popupmenu.Append(-1, "Convert to .esp") else: item = self.popupmenu.Append(-1, "Convert to .esm") self.Bind(wx.EVT_MENU, self.OnConvert, item) self.tree.Bind(wx.EVT_CONTEXT_MENU, self.OnShowPopup) def OnShowPopup(self, event): pos = event.GetPosition() pos = self.split.ScreenToClient(pos) self.tree.PopupMenu(self.popupmenu, pos) def OnSelChanged(self, event): item = event.GetItem() self.text.SetValue(self.tree.GetItemText(item)) #Called when the tree is expanded, creates child nodes def OnItemExpanding(self, event): self.addItem(event.GetItem()) #Deletes the child nodes when the tree is collapsed def OnItemCollapsed(self, event): self.tree.DeleteChildren(event.GetItem()) #Creates child nodes when given the parent node, and a list def addItem(self, parent): if self.tree.GetItemText(parent) == self.filename: if self.Changed == False: for node in self.FileObj.GRUPS: #node.scan() Sorted = Sorter.Sorter(node.TokenList) Sorted.run() newItem = self.tree.AppendItem(parent, str(Sorted.TreeList[0])) self.tree.SetItemImage(newItem, self.fldridx, wx.TreeItemIcon_Normal) self.tree.SetItemImage(newItem, self.fldropenidx, wx.TreeItemIcon_Expanded) self.tree.SetItemPyData(newItem, Sorted.TreeList[0]) self.tree.SetItemHasChildren(newItem, True) self.TreeList += Sorted.TreeList self.Changed = True else: for node in self.TreeList: newItem = self.tree.AppendItem(parent, str(node)) self.tree.SetItemImage(newItem, self.fldridx, wx.TreeItemIcon_Normal) self.tree.SetItemImage(newItem, self.fldropenidx, wx.TreeItemIcon_Expanded) self.tree.SetItemPyData(newItem, node) self.tree.SetItemHasChildren(newItem, True) else: tree = self.tree.GetItemPyData(parent) if tree.right != []: for node in tree.right: newItem = self.tree.AppendItem(parent, str(node)) if (node.right == []) and (node.left == []): self.tree.SetItemImage(newItem, self.fileidx, wx.TreeItemIcon_Normal) self.tree.SetItemPyData(newItem, node) self.tree.SetItemHasChildren(newItem, False) else: self.tree.SetItemImage(newItem, self.fldridx, wx.TreeItemIcon_Normal) self.tree.SetItemImage(newItem, self.fldropenidx, wx.TreeItemIcon_Expanded) self.tree.SetItemPyData(newItem, node) self.tree.SetItemHasChildren(newItem, True) elif tree.left != []: for node in tree.left: newItem = self.tree.AppendItem(parent, str(node)) self.tree.SetItemImage(newItem, self.fileidx, wx.TreeItemIcon_Normal) self.tree.SetItemPyData(newItem, node) self.tree.SetItemHasChildren(newItem, False) else: pass def OnSave(self, e): """Save a file""" f = open(self.name, 'wb') f.write('') f.close() f = open(self.name, 'ab') for token, data in self.FileObj.TES4.TokenList: f.write(token) f.write(data) if self.Changed == False: for node in self.FileObj.GRUPS: f.write(node.expr.read()) else: self.BodySave(f, self.TreeList) f.close() d = wx.MessageDialog(self, "Save Complete", "Finished", wx.OK) d.ShowModal() d.Destroy() def OnSaveAs(self,e): # Save away the edited text # Open the file, do an RU sure check for an overwrite! dlg = wx.FileDialog(self, "Choose a file", self.dirname, "", "esp/esm files | *esp;*.esm", wx.SAVE | wx.OVERWRITE_PROMPT) if dlg.ShowModal() == wx.ID_OK: # Open the file for write, write, close f=open(dlg.GetPath(),'w') f.close() f=open(dlg.GetPath(),'ab') for token, data in self.FileObj.TES4.TokenList: f.write(token) f.write(data) if self.Changed == False: for node in self.FileObj.GRUPS: f.write(node.expr.read()) else: self.BodySave(f, self.TreeList) f.close() d = wx.MessageDialog(self, "Save Complete", "Finished", wx.OK) d.ShowModal() d.Destroy() # Get rid of the dialog to keep things tidy dlg.Destroy() def BodySave(self, File, root): for node in root: node.save(File) if node.right == []: pass else: self.BodySave(File, node.right) if node.left == []: pass else: self.BodySave(File, node.left) def OnConvert(self, event): (shortname, ext) = os.path.splitext(self.filename) text = cStringIO.StringIO(self.FileObj.TES4.TokenList[0][1]) if ext == ".esm": new = text.read(4) + '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00' filename = shortname + ".esp" else: new = text.read(4) + '\x01\x00\x00\x00\x96\x00\x00\x00\x00\x00\x00\x00' filename = shortname + ".esm" self.FileObj.TES4.TokenList[0][1] = new f = open(os.path.join(self.dirname, filename), 'w') f.close() f = open(os.path.join(self.dirname, filename), 'ab') for token, data in self.FileObj.TES4.TokenList: f.write(token) f.write(data) if self.Changed == False: for node in self.FileObj.GRUPS: node.expr.seek(0) f.write(node.expr.read()) else: self.BodySave(f, self.TreeList) f.close() d = wx.MessageDialog(self, "Conversion Complete", "Finished", wx.OK) d.ShowModal() d.Destroy() class MyApp(wx.App): def OnInit(self): bmp = wx.Image("splash.png").ConvertToBitmap() wx.SplashScreen(bmp, wx.SPLASH_CENTRE_ON_SCREEN | wx.SPLASH_TIMEOUT, 1000, None, -1) wx.Yield() frame = ParentFrame(None) frame.Show(True) self.SetTopWindow(frame) return True if __name__ == '__main__': app = MyApp(0) app.MainLoop()  TES4scanner.py import re import StringIO class Scanner(): def __init__(self, expr): self.TokenList = [] self.scanner = re.compile(r""" (?P<TES4>TES4) | (?P<HEDR>HEDR) | (?P<CNAM>CNAM) | (?P<SNAM>SNAM) | (?P<MAST>MAST) | (?P<DATAkeyword>DATA) | (?P<GRUP>GRUP) | (?P<DATA>(.)*?((?=HEDR)|(?=CNAM)|(?=SNAM)| (?=MAST)|(?=DATA)|(?=GRUP))) """,re.VERBOSE | re.DOTALL) self.expr = StringIO.StringIO(expr) self.pos = 0 def scan(self): lastType = None seekPos = self.pos print "Token stream for a TES4 header:" print "-------------------------------" while 1: self.expr.seek(self.pos) x = self.scanner.match(self.expr.read(1000)) if not x: break else: pass if x.lastindex == 1: Type = 'TES4' elif x.lastindex == 2: Type = 'HEDR' elif x.lastindex == 3: Type = 'CNAM' elif x.lastindex == 4: Type = 'SNAM' elif x.lastindex == 5: Type = 'MAST' elif x.lastindex == 6: Type = 'DATA' elif x.lastindex == 7: Type = 'GRUP' break elif x.lastindex == 8: Type = 'data' else: pass self.expr.seek(seekPos) data = self.expr.read(x.end()-x.start()) if lastType == Type: if Type == 'data': self.TokenList[len(self.TokenList) - 1][1] += data else: pass else: if Type == 'data': self.TokenList[len(self.TokenList) - 1].append(data) lastType = Type else: self.TokenList.append([Type]) lastType = Type if __name__ == '__main__': print "%s | %s | %s" % (Type, x.start(), x.end()) self.pos += x.end() seekPos += x.end()-x.start() print "-------------------------------" self.expr.seek(0) self.HEDR = self.expr.read(self.pos) if __name__ == '__main__': f = open("H:/Desktop/Living Economy.esp", 'rb') #C:/Program Files/Bethesda Softworks/Oblivion/Data/test.espC:/Program Files/Bethesda Softworks/Oblivion/Data/Unofficial Oblivion Patch.esp TES4 = Scanner(f.read()) f.close() TES4.scan() #def loop(): # expr = raw_input("What would you like to scan? ") # thing = Scanner(expr) # thing.scan() # loop() #loop()  GRUPscanner.py import cStringIO import struct class Scanner(): def __init__(self, expr, pos=0): self.TokenList = [] #self.scanner = re.compile(r""" # (?P<GRUP>GRUP) | (?P<GMST>GMST) | (?P<GLOB>GLOB) | # (?P<CLAS>CLAS) | (?P<FACT>FACT) | (?P<HAIR>HAIR) | # (?P<EYES>EYES) | (?P<RACE>RACE) | (?P<SOUN>SOUN) | # (?P<SKIL>SKIL) | (?P<MGEF>MGEF) | (?P<SCPT>SCPT) | # (?P<LTEX>LTEX) | (?P<ENCH>ENCH) | (?P<SPEL>SPEL) | # (?P<BSGN>BSGN) | (?P<ACTI>ACTI) | (?P<APPA>APPA) | # (?P<ARMO>ARMO) | (?P<BOOK>BOOK) | (?P<CLOT>CLOT) | # (?P<CONT>CONT) | (?P<DOOR>DOOR) | (?P<INGR>INGR) | # (?P<LIGH>LIGH) | (?P<MISC>MISC) | (?P<STAT>STAT) | # (?P<GRAS>GRAS) | (?P<TREE>TREE) | (?P<FLOR>FLOR) | # (?P<FURN>FURN) | (?P<WEAP>WEAP) | (?P<AMMO>AMMO) | # (?P<NPC_>NPC_) | (?P<CREA>CREA) | (?P<LVLC>LVLC) | # (?P<SLGM>SLGM) | (?P<KEYM>KEYM) | (?P<ALCH>ALCH) | # (?P<SBSP>SBSP) | (?P<SGST>SGST) | (?P<LVLI>LVLI) | # (?P<WTHR>WTHR) | (?P<CLMT>CLMT) | (?P<REGN>REGN) | # (?P<CELL>CELL) | (?P<WRLD>WRLD) | (?P<DIAL>DIAL) | # (?P<QUST>QUST) | (?P<IDLE>IDLE) | (?P<PACK>PACK) | # (?P<CSTY>CSTY) | (?P<LSCR>LSCR) | (?P<LVSP>LVSP) | # (?P<ANIO>ANIO) | (?P<WATR>WATR) | (?P<EFSH>EFSH) # """,re.VERBOSE | re.DOTALL) self.expr = cStringIO.StringIO(expr) self.seekPos = pos def scan(self): print "Token stream for a GRUP records:" print "-------------------------------" append = self.TokenList.append seek = self.expr.seek read = self.expr.read while 1: seek(self.seekPos) Type = read(4) #self.expr.seek(self.seekPos + 4) if Type == 'GRUP': length = 16 data = read(length) else: length = read(4) length = struct.unpack('l', length) length = length[0] + 16 seek(self.seekPos + 4) data = read(length) append([Type, data]) if __name__ == '__main__': print "%s" % Type self.seekPos += 4 + length seek(0) if self.seekPos >= len(read()): break print "-------------------------------" if __name__ == '__main__': f = open("C:/Program Files/Bethesda Softworks/Oblivion/Data/test.esp", 'rb') #C:/Program Files/Bethesda Softworks/Oblivion/Data/test.esp GRUP = Scanner(f.read(), 85) f.close() GRUP.scan()  test.py import struct import GRUPscanner import re import cStringIO import TES4scanner class File(): def __init__(self, filename): self.filename = filename self.GRUPS = [] def parseRecord(self): f = open(self.filename, "rb") seek = f.seek read = f.read #append = self.GRUPS.append self.TES4 = TES4scanner.Scanner(f.read()) self.TES4.scan() seek(0) grupMatch = re.compile('GRUP') allMatches = grupMatch.finditer(f.read()) [self.parseMatch(seek, read, match) for match in allMatches] """for match in allMatches: seek(match.end() + 8) flag = struct.unpack('l', read(4)) if flag[0] == 000000: #-find the length of the GRUP record seek(match.end()) length = struct.unpack('l', read(4)) #-Once the length is found, we read it into the list seek(match.start()) append(GRUPscanner.Scanner(read(length[0]))) else: pass""" f.close() print "sucess!" def parseMatch(self, seek, read, match): #seek = f.seek #read = f.read append = self.GRUPS.append seek(match.end() + 8) flag = struct.unpack('l', read(4)) if flag[0] == 000000: #-find the length of the GRUP record seek(match.end()) length = struct.unpack('l', read(4)) #-Once the length is found, we read it into the list seek(match.start()) append(GRUPscanner.Scanner(read(length[0]))) else: pass if __name__ == '__main__': obj = File("H:/Desktop/Living Economy.esp") obj.parseRecord()  Sorter.py import struct import cStringIO class BinaryTree(): def __init__(self, name, data, flag=None, parent=None): self.name = name self.data = data self.left = [] self.right = [] self.parent = parent self.flag = flag def __str__(self): if len(self.data) == 16: return "%s: %s" % (self.name, struct.unpack('l4sll', self.data)) else: return "%s" % self.name def __len__(self): return len(data) def save(self, File): File.write(self.name) File.write(self.data) class GMST(BinaryTree): def __init__(self, name, data, parent): BinaryTree.__init__(self, name, data, None, parent) self.data = cStringIO.StringIO(data) self.hedr = self.data.read(16) self.edid = self.parse() self.fnam = "" def __str__(self): return "%s" % self.edid[2] def OnDialog(self): self.fnam = self.parse() def parse(self): title = self.data.read(4) length = struct.unpack('H', self.data.read(2)) info = self.data.read(length[0]) x = [] x.append(title) x.append(length) x.append(info) return x def save(self, File): self.data.seek(0) File.write(self.name) File.write(self.data.read()) class GLOB(BinaryTree): def __init__(self, name, data, parent): BinaryTree.__init__(self, name, data, None, parent) self.data = cStringIO.StringIO(data) self.hedr = self.data.read(16) self.edid = self.parse() self.fnam = "" self.fltv = "" def __str__(self): return "%s" % self.edid[2] def OnDialog(self): self.fnam = self.parse() self.fltv = self.parse() def parse(self): title = self.data.read(4) length = struct.unpack('H', self.data.read(2)) info = self.data.read(length[0]) x = [] x.append(title) x.append(length) x.append(info) return x def save(self, File): self.data.seek(0) File.write(self.name) File.write(self.data.read()) #------------------------------------------------------------------------------- class Sorter(): def __init__(self, TokenList): self.TreeList = [] self.tree = None self.TokenList = TokenList self.GRUPdict = {0: lambda x, y, flag: self.Ctree(x, y, flag), 1: lambda x, y, flag: self.Ctree(x, y, flag, self.tree), 2: lambda x, y, flag: self.Ctree(x, y, flag, self.tree), 3: lambda x, y, flag: self.Ctree(x, y, flag, self.tree), 4: lambda x, y, flag: self.Ctree(x, y, flag, self.tree), 5: lambda x, y, flag: self.Ctree(x, y, flag, self.tree), 6: lambda x, y, flag: self.Ctree(x, y, flag, self.tree), 7: lambda x, y, flag: self.Ctree(x, y, flag, self.tree), 8: lambda x, y, flag: self.Ctree(x, y, flag, self.tree), #TODO: Fix 9: lambda x, y, flag: self.Ctree(x, y, flag, self.tree), #TODO: Fix 10: lambda x, y, flag: self.Ctree(x, y, flag, self.tree)} #TODO: Fix def run(self): self.x = [self.JumpTo(token, data) for token, data in self.TokenList] def Ctree(self, name, data, flag, parent=None): if parent == None: newTree = BinaryTree(name, data, flag) self.TreeList.append(newTree) self.tree = newTree else: if (flag == 1) | (flag == 2): self.tree = self.TreeList[len(self.TreeList) - 1] newTree = BinaryTree(name, data, flag, parent) self.tree.right.append(newTree) self.tree = newTree elif flag == 4: self.tree = self.TreeList[len(self.TreeList) - 1] self.tree = self.tree.right[len(self.tree.right) - 1] newTree = BinaryTree(name, data, flag, parent) self.tree.right.append(newTree) self.tree = newTree elif flag == 7: newTree = BinaryTree(name, data, flag, parent) self.tree.right.append(newTree) elif (flag == 8) | (flag == 9) | (flag == 10): newTree = BinaryTree(name, data, flag, parent) self.tree.right.append(newTree) else: newTree = BinaryTree(name, data, flag, parent) self.tree.right.append(newTree) self.tree = newTree def JumpTo(self, token, data): if token == 'GRUP': self.GRUPhandler(token, data) elif token == 'CELL': self.CELLhandler(token, data) elif token == 'ROAD': self.ROADhandler(token, data) elif token == 'WRLD': self.WRLDhandler(token, data) else: self.GenericHandler(token, data) def GRUPhandler(self, token, data): string = cStringIO.StringIO(data) string.seek(8) flag = struct.unpack('l', str(string.read(4))) self.GRUPdict[flag[0]](token, data, flag[0]) def CELLhandler(self, token, data): if self.tree.flag == 6: self.tree = self.tree.parent newTree = BinaryTree(token, data) self.tree.right.append(newTree) def ROADhandler(self, token, data): newTree = BinaryTree(token, data) self.tree.right.append(newTree) def WRLDhandler(self, token, data): self.tree = self.TreeList[len(self.TreeList) - 1] newTree = BinaryTree(token, data) self.tree.right.append(newTree) def DIALhandler(self, token, data): self.tree = self.TreeList[len(self.TreeList) - 1] newTree = BinaryTree(token, data) self.tree.right.append(newTree) def GenericHandler(self, token, data): if (token == 'REFR') | (token == 'ACHR') | (token == 'ACRE') | (token == 'LAND') | (token == 'PGRD'): newTree = BinaryTree(token, data, self.tree) self.tree.right[len(self.tree.right) - 1].left.append(newTree) elif token == 'GMST': newTree = GMST(token, data, self.tree) self.tree.left.append(newTree) elif token == 'GLOB': newTree = GLOB(token, data, self.tree) self.tree.left.append(newTree) else: newTree = BinaryTree(token, data, self.tree) self.tree.left.append(newTree) #-------------------------------------------------------------------------------- if __name__ == '__main__': TokenList = [['GRUP', '\x15\x01\x00\x00CELL\x00\x00\x00\x00\x1a3\x00\x00'], ['GRUP', '\x01\x01\x00\x00\t\x00\x00\x00\x02\x00\x00\x00\x1a3\x00\x00'], ['GRUP', '\xed\x00\x00\x00\x04\x00\x00\x00\x03\x00\x00\x00\x1a3\x00\x00'], ['CELL', '\x00\x00\x00\x00\x00\x00=\\\x01\x00\x00\x00\x00\x00EDID\x0e\x00AbandonedMine\x00FULL\x0f\x00Abandoned Mine\x00DATA\x01\x00AXCLL$\x00\x16\x13\x10\x00\x00\x00\x00\x00\x19\x14\x0b\x00\x00\x00\x00\x00\x00\x00\xfaD\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80?\x00\x00\xfaDXCMT\x01\x00\x02GR'],
['GRUP', 'd\x00\x00\x00=\\\x01\x00\x06\x00\x00\x00\x1a3\x00\x00'],
['GRUP', 'P\x00\x00\x00=\\\x01\x00\t\x00\x00\x00\x1a3\x00\x00'],
['REFR', '\x00\x00\x00\x00\x00\x00\xe6\x0c\x00\x01\x1a3\x00\x00NAME\x04\x00zw\x01\x00DATA\x18\x00\x8f;\xffDL\xcf\x00E^\x9a\xfdE\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00']]

main = Sorter(TokenList)
main.run()


Now, I'm postive that the slowness is coming from either test.py or GRUPscanner.py, but most likely GRUPscanner.py. This means its related to the cStringIO module, and the many appends I do. Could anybody help me fix this so it doesn't take 5 minutes to load a 10mb, or is my only hope writing a c extension for python. Note, that I've already tried solutions such as Pyrex which have not helped. Also, I have done many goole searches, but haven't found anything helpful.

##### Share on other sites
I have, and I determined that the slow down usually comes from test.py or GRUPscanner.py. However, I have no idea how to optimize it, and google was very unhelpful. Which is why I came here, I was wondering if anybody here knew of some good omptimizations in python.

##### Share on other sites
The profiler can tell you exactly which functions are slow, not just which files. There's an awful lot of code there, and although I'm actually quite good at optimising Python code, I don't have the time on my hands to read through all of that.

That seek(0) looks pretty pointless though.

##### Share on other sites
Python probably isn't the problem here. It looks like most of what you are doing is very high level operations, which python handles very well since its implementations of those operations are fast.

I think the problem is excessive I/O. Read the file once and then work with the string that is returned. For large files, it might be better to read the file and process it incrementally, so that you don't need to hold the entire file in memory. Your code currently reads the entire file, performs pattern matching on it, and then jumps around the file reading off the matches for parsing. It would be better to read the file once and then parse the matches by slicing that string, instead of going back to the file again to read the matches.

##### Share on other sites
I found out what the problem was, and now it loads the 10mb file in less than 5 seconds. The problem was that in the GRUPscanner.py module, at the end of the while loop, I was checking the length of the file with len(file.read( )) by declaring a local variable such as len = len(file.read()) and checking against that local variable, I drastically improved the speed.

##### Share on other sites
Ah, yes. Re-reading the entire file once for every handful of bytes probably isn't the most efficient method.

1. 1
2. 2
3. 3
4. 4
Rutin
11
5. 5

• 12
• 19
• 10
• 14
• 10
• ### Forum Statistics

• Total Topics
632666
• Total Posts
3007723
• ### Who's Online (See full list)

There are no registered users currently online

×