Jump to content
  • Advertisement
Sign in to follow this  
ForD

optimizations in python

This topic is 4229 days old which is more than the 365 day threshold we allow for new replies. Please post a new topic.

If you intended to correct an error in the post then please contact us.

Recommended Posts

Currently I'm in the porcess of writing a program in python that opens and edits a modfile for a certain game. These files can be anywhere from 1mb to 300mb. However, my program isn't that fast, it takes around 5 minutes to load something 10mb in size. Before I elaborate further, here's my sourcecode: GUI.py
import wx
import wx.aui
import os
import cStringIO
import test
import Sorter

#-------------------
def GetMondrianData():
    return '\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00 \x00\x00\x00 \x08\x06\x00\x00\x00szz\xf4\x00\x00\x00\x04sBIT\x08\x08\x08\x08|\x08d\x88\x00\x00\x00qIDATX\x85\xed\xd6;\n\x800\x10E\xd1{\xc5\x8d\xb9r\x97\x16\x0b\xad$\x8a\x82:\x16o\xda\x84pB2\x1f\x81Fa\x8c\x9c\x08\x04Z{\xcf\xa72\xbcv\xfa\xc5\x08 \x80r\x80\xfc\xa2\x0e\x1c\xe4\xba\xfaX\x1d\xd0\xde]S\x07\x02\xd8>\xe1wa-`\x9fQ\xe9\x86\x01\x04\x10\x00\\(Dk\x1b-\x04\xdc\x1d\x07\x14\x98;\x0bS\x7f\x7f\xf9\x13\x04\x10@\xf9X\xbe\x00\xc9 \x14K\xc1<={\x00\x00\x00\x00IEND\xaeB`\x82' 


def GetMondrianBitmap():
    return wx.BitmapFromImage(GetMondrianImage())


def GetMondrianImage():
    import cStringIO
    stream = cStringIO.StringIO(GetMondrianData())
    return wx.ImageFromStream(stream)


def GetMondrianIcon():
    icon = wx.EmptyIcon()
    icon.CopyFromBitmap(GetMondrianBitmap())
    return icon
#--------------------------------
class ParentFrame(wx.aui.AuiMDIParentFrame):
    def __init__(self, parent):
        wx.aui.AuiMDIParentFrame.__init__(self, parent, -1,
                                          title="Tes4Editor",
                                          size=(640,480),
                                          style=wx.DEFAULT_FRAME_style)
        self.SetIcon(GetMondrianIcon())
        mb = self.MakeMenuBar()
        self.SetMenuBar(mb)
        self.CreateStatusBar()
        


    def MakeMenuBar(self):
        mb = wx.MenuBar()
        menu = wx.Menu()
        item = menu.Append(-1, "Open...\tCtrl-O")
        self.Bind(wx.EVT_MENU, self.OnOpen, item)
        item = menu.Append(-1, "&Quit\tCtrl-Q")
        self.Bind(wx.EVT_MENU, self.OnDoClose, item)
        menu2 = wx.Menu()
        item = menu2.Append(-1, "About\tCtrl-A")
        self.Bind(wx.EVT_MENU, self.OnAbout, item)
        mb.Append(menu, "&File") 
        mb.Append(menu2, "&Help")
        return mb


    def OnOpen(self, e):
        """ Open a file"""
        self.dirname = ''
        dlg = wx.FileDialog(self, "Choose a file", self.dirname, "", "esp/esm files | *esp;*.esm", 
wx.OPEN)
        if dlg.ShowModal() == wx.ID_OK:
            self.filename = dlg.GetFilename()
            self.dirname = dlg.GetDirectory()
            filename = os.path.join(self.dirname,self.filename)
            child = ChildFrame(self, self.filename, self.dirname, filename)
            child.Show()
        dlg.Destroy()
    def OnAbout(self, e):
        msg = "TES4++ - Utitlity for low level \nediting of esp/esm files."
        d = wx.MessageDialog(self, msg, "TES4++", wx.OK | wx.ICON_INFORMATION)
        d.ShowModal()
        d.Destroy()
                  
    def OnDoClose(self, evt):
        self.Close()

class ChildFrame(wx.aui.AuiMDIChildFrame):
    def __init__(self, parent, filename="test.esp", dirname="C:/Program Files/Bethesda Softworks/Oblivion/Data", name="C:/Program Files/Bethesda Softworks/Oblivion/Data/test.esp"):
        wx.aui.AuiMDIChildFrame.__init__(self, parent, -1,
                                         title = filename)
        self.filename = filename
        self.dirname = dirname
        self.name = name

        mb = parent.MakeMenuBar()
        menu = wx.Menu()
        item = menu.Append(-1, "&Save\tCtrl-S")
        self.Bind(wx.EVT_MENU, self.OnSave, item)
        item = menu.Append(-1, "&Save As...")
        self.Bind(wx.EVT_MENU, self.OnSaveAs, item)
        mb.Append(menu, self.filename)
        self.SetMenuBar(mb)

        self.FileObj = test.File(self.name)
        self.FileObj.parseRecord()
        for node in self.FileObj.GRUPS:
            node.scan()
        self.TreeList = []
        self.Changed = False
        self.split = wx.SplitterWindow(self, -1)

        #------Tree here----------------------------------
        il = wx.ImageList(16,16)

        self.fldridx = il.Add(
            wx.ArtProvider.GetBitmap(wx.ART_FOLDER,
                    wx.ART_OTHER, (16,16)))
        self.fldropenidx = il.Add(
            wx.ArtProvider.GetBitmap(wx.ART_FILE_OPEN,
                    wx.ART_OTHER, (16,16)))
        self.fileidx = il.Add(
            wx.ArtProvider.GetBitmap(wx.ART_NORMAL_FILE,
                    wx.ART_OTHER, (16,16)))

        self.tree = wx.TreeCtrl(self.split, 1, wx.DefaultPosition, (-1,-1),
                                wx.TR_HAS_BUTTONS)
        self.root = self.tree.AddRoot(self.filename)

        self.tree.AssignImageList(il)
        self.tree.SetItemImage(self.root, self.fldridx,
                               wx.TreeItemIcon_Normal)
        self.tree.SetItemImage(self.root, self.fldropenidx,
                               wx.TreeItemIcon_Expanded)
        self.tree.SetItemPyData(self.root, self.FileObj.GRUPS)
        self.tree.SetItemHasChildren(self.root, True)

        #Bind some tree events
        #self.Bind(wx.EVT_TREE_SEL_CHANGED, self.OnSelChanged, self.tree)
        self.Bind(wx.EVT_TREE_ITEM_EXPANDING, self.OnItemExpanding, self.tree)
        self.Bind(wx.EVT_TREE_ITEM_COLLAPSED, self.OnItemCollapsed, self.tree)

        #-----------------------------------------------------
        self.panel = wx.Panel(self.split, -1)
        self.split.SplitVertically(self.tree, self.panel, 200)

        sizer = wx.BoxSizer()
        sizer.Add(self.split, 1, wx.EXPAND)
        self.SetSizer(sizer)

        wx.CallAfter(self.Layout)

        (shortname, ext) = os.path.splitext(self.filename)
        
        self.popupmenu = wx.Menu()
        item = self.popupmenu.Append(-1, "Save")
        self.Bind(wx.EVT_MENU, self.OnSave, item)
        item = self.popupmenu.Append(-1, "Save As...")
        self.Bind(wx.EVT_MENU, self.OnSaveAs, item)
        if ext == ".esm":
            item = self.popupmenu.Append(-1, "Convert to .esp")
        else:
            item = self.popupmenu.Append(-1, "Convert to .esm")
        self.Bind(wx.EVT_MENU, self.OnConvert, item)
        self.tree.Bind(wx.EVT_CONTEXT_MENU, self.OnShowPopup)

    def OnShowPopup(self, event):
        pos = event.GetPosition()
        pos = self.split.ScreenToClient(pos)
        self.tree.PopupMenu(self.popupmenu, pos)

    def OnSelChanged(self, event):
        item =  event.GetItem()
        self.text.SetValue(self.tree.GetItemText(item))

    #Called when the tree is expanded, creates child nodes
    def OnItemExpanding(self, event):
        self.addItem(event.GetItem())


    #Deletes the child nodes when the tree is collapsed
    def OnItemCollapsed(self, event):
        self.tree.DeleteChildren(event.GetItem())

    #Creates child nodes when given the parent node, and a list
    def addItem(self, parent):
        if self.tree.GetItemText(parent) == self.filename:
            if self.Changed == False:
                for node in self.FileObj.GRUPS:
                    #node.scan()
                    Sorted = Sorter.Sorter(node.TokenList)
                    Sorted.run()
                    newItem = self.tree.AppendItem(parent, str(Sorted.TreeList[0]))
                    self.tree.SetItemImage(newItem, self.fldridx,
                                                wx.TreeItemIcon_Normal)
                    self.tree.SetItemImage(newItem, self.fldropenidx,
                                                wx.TreeItemIcon_Expanded)
                    self.tree.SetItemPyData(newItem, Sorted.TreeList[0])
                    self.tree.SetItemHasChildren(newItem, True)
                    self.TreeList += Sorted.TreeList
                self.Changed = True
            else:
                for node in self.TreeList:
                    newItem = self.tree.AppendItem(parent, str(node))
                    self.tree.SetItemImage(newItem, self.fldridx,
                                           wx.TreeItemIcon_Normal)
                    self.tree.SetItemImage(newItem, self.fldropenidx,
                                           wx.TreeItemIcon_Expanded)
                    self.tree.SetItemPyData(newItem, node)
                    self.tree.SetItemHasChildren(newItem, True)
        else:
            tree = self.tree.GetItemPyData(parent)
            if tree.right != []:
                for node in tree.right:
                    newItem = self.tree.AppendItem(parent, str(node))
                    if (node.right == []) and (node.left == []):
                        self.tree.SetItemImage(newItem, self.fileidx,
                                               wx.TreeItemIcon_Normal)
                        self.tree.SetItemPyData(newItem, node)
                        self.tree.SetItemHasChildren(newItem, False)
                    else:
                        self.tree.SetItemImage(newItem, self.fldridx,
                                               wx.TreeItemIcon_Normal)
                        self.tree.SetItemImage(newItem, self.fldropenidx,
                                               wx.TreeItemIcon_Expanded)
                        self.tree.SetItemPyData(newItem, node)
                        self.tree.SetItemHasChildren(newItem, True)
            elif tree.left != []:
                for node in tree.left:
                    newItem = self.tree.AppendItem(parent, str(node))
                    self.tree.SetItemImage(newItem, self.fileidx,
                                            wx.TreeItemIcon_Normal)
                    self.tree.SetItemPyData(newItem, node)
                    self.tree.SetItemHasChildren(newItem, False)
            else:
                pass
        

    def OnSave(self, e):
        """Save a file"""
        f = open(self.name, 'wb')
        f.write('')
        f.close()
        f = open(self.name, 'ab')
        for token, data in self.FileObj.TES4.TokenList:
            f.write(token)
            f.write(data)
        if self.Changed == False:
            for node in self.FileObj.GRUPS:
                f.write(node.expr.read())
        else:
            self.BodySave(f, self.TreeList)
        f.close()
        d = wx.MessageDialog(self, "Save Complete", "Finished", wx.OK)
        d.ShowModal()
        d.Destroy()

    def OnSaveAs(self,e):
    # Save away the edited text
    # Open the file, do an RU sure check for an overwrite!
        dlg = wx.FileDialog(self, "Choose a file", self.dirname, "", "esp/esm files | *esp;*.esm",         wx.SAVE | wx.OVERWRITE_PROMPT)
        if dlg.ShowModal() == wx.ID_OK:
        # Open the file for write, write, close
            f=open(dlg.GetPath(),'w')
            f.close()
            f=open(dlg.GetPath(),'ab')
            for token, data in self.FileObj.TES4.TokenList:
                f.write(token)
                f.write(data)
            if self.Changed == False:
                for node in self.FileObj.GRUPS:
                    f.write(node.expr.read())
            else:
                self.BodySave(f, self.TreeList)
            f.close()
            d = wx.MessageDialog(self, "Save Complete", "Finished", wx.OK)
            d.ShowModal()
            d.Destroy()
    # Get rid of the dialog to keep things tidy
        dlg.Destroy()
        
    def BodySave(self, File, root):
        for node in root:
            node.save(File)
            if node.right == []:
                pass
            else:
                self.BodySave(File, node.right)
            if node.left == []:
                pass
            else:
                self.BodySave(File, node.left)

    def OnConvert(self, event):
        (shortname, ext) = os.path.splitext(self.filename)
        text = cStringIO.StringIO(self.FileObj.TES4.TokenList[0][1])
        if ext == ".esm":
            new = text.read(4) + '\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
            filename = shortname + ".esp"
        else:
            new = text.read(4) + '\x01\x00\x00\x00\x96\x00\x00\x00\x00\x00\x00\x00'
            filename = shortname + ".esm"
        self.FileObj.TES4.TokenList[0][1] = new    
        f = open(os.path.join(self.dirname, filename), 'w')
        f.close()
        f = open(os.path.join(self.dirname, filename), 'ab')
        for token, data in self.FileObj.TES4.TokenList:
            f.write(token)
            f.write(data)
        if self.Changed == False:
            for node in self.FileObj.GRUPS:
                node.expr.seek(0)
                f.write(node.expr.read())
        else:
            self.BodySave(f, self.TreeList)
        f.close()
        d = wx.MessageDialog(self, "Conversion Complete", "Finished", wx.OK)
        d.ShowModal()
        d.Destroy()

class MyApp(wx.App):
    def OnInit(self):
        bmp = wx.Image("splash.png").ConvertToBitmap()
        wx.SplashScreen(bmp, wx.SPLASH_CENTRE_ON_SCREEN |
                        wx.SPLASH_TIMEOUT,
                        1000, None, -1)
        wx.Yield()

        frame = ParentFrame(None)
        frame.Show(True)
        self.SetTopWindow(frame)
        return True

if __name__ == '__main__':
    app = MyApp(0)
    app.MainLoop()



TES4scanner.py
import re
import StringIO
class Scanner():
    def __init__(self, expr):   
        self.TokenList = [] 
        self.scanner = re.compile(r"""
                        (?P<TES4>TES4) | 
                        (?P<HEDR>HEDR) |
                        (?P<CNAM>CNAM) |
                        (?P<SNAM>SNAM) |
                        (?P<MAST>MAST) |
                        (?P<DATAkeyword>DATA) |
                        (?P<GRUP>GRUP) |
                        (?P<DATA>(.)*?((?=HEDR)|(?=CNAM)|(?=SNAM)|
                                        (?=MAST)|(?=DATA)|(?=GRUP)))
                         """,re.VERBOSE | re.DOTALL)
        self.expr = StringIO.StringIO(expr)

        self.pos = 0

    def scan(self):
        lastType = None
        seekPos = self.pos
        print "Token stream for a TES4 header:"
        print "-------------------------------"
        while 1:
            self.expr.seek(self.pos)
            x = self.scanner.match(self.expr.read(1000))
            if not x:
                break
            else:
                pass
    
            if x.lastindex == 1:
                Type = 'TES4'
            elif x.lastindex == 2:
                Type = 'HEDR'
            elif x.lastindex == 3:
                Type = 'CNAM'
            elif x.lastindex == 4:
                Type = 'SNAM'
            elif x.lastindex == 5:
                Type = 'MAST'
            elif x.lastindex == 6:
                Type = 'DATA'
            elif x.lastindex == 7:
                Type = 'GRUP'
                break
            elif x.lastindex == 8: 
                Type = 'data'
            else:
                pass

            self.expr.seek(seekPos)
            data = self.expr.read(x.end()-x.start())
            if lastType == Type:
                if Type == 'data':
                    self.TokenList[len(self.TokenList) - 1][1] += data
                else:
                    pass
            else:
                if Type == 'data':
                    self.TokenList[len(self.TokenList) - 1].append(data)
                    lastType = Type
                else:
                    self.TokenList.append([Type])
                    lastType = Type
            if __name__ == '__main__':
                print "%s | %s | %s" % (Type, x.start(), x.end())
            self.pos += x.end()
            seekPos += x.end()-x.start()
        print "-------------------------------"
        self.expr.seek(0)
        self.HEDR = self.expr.read(self.pos)

if __name__ == '__main__':
    f = open("H:/Desktop/Living Economy.esp", 'rb') #C:/Program Files/Bethesda Softworks/Oblivion/Data/test.espC:/Program Files/Bethesda Softworks/Oblivion/Data/Unofficial Oblivion Patch.esp
    TES4 = Scanner(f.read())
    f.close()
    TES4.scan()
    #def loop():
    #    expr = raw_input("What would you like to scan? ") 
    #    thing = Scanner(expr)
    #    thing.scan()
    #    loop()
    #loop()



GRUPscanner.py
import cStringIO
import struct
class Scanner():
    def __init__(self, expr, pos=0):   
        self.TokenList = [] 
        #self.scanner = re.compile(r"""
        #                (?P<GRUP>GRUP) | (?P<GMST>GMST) | (?P<GLOB>GLOB) |
        #                (?P<CLAS>CLAS) | (?P<FACT>FACT) | (?P<HAIR>HAIR) |
        #                (?P<EYES>EYES) | (?P<RACE>RACE) | (?P<SOUN>SOUN) |
        #                (?P<SKIL>SKIL) | (?P<MGEF>MGEF) | (?P<SCPT>SCPT) |
        #                (?P<LTEX>LTEX) | (?P<ENCH>ENCH) | (?P<SPEL>SPEL) |
        #                (?P<BSGN>BSGN) | (?P<ACTI>ACTI) | (?P<APPA>APPA) |
        #                (?P<ARMO>ARMO) | (?P<BOOK>BOOK) | (?P<CLOT>CLOT) |
        #                (?P<CONT>CONT) | (?P<DOOR>DOOR) | (?P<INGR>INGR) |
        #                (?P<LIGH>LIGH) | (?P<MISC>MISC) | (?P<STAT>STAT) |
        #                (?P<GRAS>GRAS) | (?P<TREE>TREE) | (?P<FLOR>FLOR) |
        #                (?P<FURN>FURN) | (?P<WEAP>WEAP) | (?P<AMMO>AMMO) |
        #                (?P<NPC_>NPC_) | (?P<CREA>CREA) | (?P<LVLC>LVLC) |
        #                (?P<SLGM>SLGM) | (?P<KEYM>KEYM) | (?P<ALCH>ALCH) |
        #                (?P<SBSP>SBSP) | (?P<SGST>SGST) | (?P<LVLI>LVLI) |
        #                (?P<WTHR>WTHR) | (?P<CLMT>CLMT) | (?P<REGN>REGN) |
        #                (?P<CELL>CELL) | (?P<WRLD>WRLD) | (?P<DIAL>DIAL) |
        #                (?P<QUST>QUST) | (?P<IDLE>IDLE) | (?P<PACK>PACK) |
        #                (?P<CSTY>CSTY) | (?P<LSCR>LSCR) | (?P<LVSP>LVSP) |
        #                (?P<ANIO>ANIO) | (?P<WATR>WATR) | (?P<EFSH>EFSH) 
        #                 """,re.VERBOSE | re.DOTALL)
        self.expr = cStringIO.StringIO(expr)

        self.seekPos = pos

    def scan(self):
        print "Token stream for a GRUP records:"
        print "-------------------------------"
        append = self.TokenList.append
        seek = self.expr.seek
        read = self.expr.read
        while 1:
            seek(self.seekPos)
            Type = read(4)
            #self.expr.seek(self.seekPos  + 4)
            if Type == 'GRUP':
                length = 16
                data = read(length)
            else:
                length = read(4)
                length = struct.unpack('l', length)
                length = length[0] + 16
                seek(self.seekPos  + 4)
                data = read(length)
            append([Type, data])
            if __name__ == '__main__':
                print "%s" % Type
            self.seekPos += 4 + length
            seek(0)
            if self.seekPos >= len(read()):
                break
        print "-------------------------------"

if __name__ == '__main__':
    f = open("C:/Program Files/Bethesda Softworks/Oblivion/Data/test.esp", 'rb') #C:/Program Files/Bethesda Softworks/Oblivion/Data/test.esp
    GRUP = Scanner(f.read(), 85)
    f.close()
    GRUP.scan()



test.py
import struct
import GRUPscanner
import re
import cStringIO
import TES4scanner

class File():
    def __init__(self, filename):
        self.filename = filename
        self.GRUPS = []

    def parseRecord(self):
        f = open(self.filename, "rb")
        seek = f.seek
        read = f.read
        #append = self.GRUPS.append
        self.TES4 = TES4scanner.Scanner(f.read())
        self.TES4.scan()
        seek(0)
        grupMatch = re.compile('GRUP')
        allMatches = grupMatch.finditer(f.read())
        [self.parseMatch(seek, read, match) for match in allMatches]
        """for match in allMatches:
            seek(match.end() + 8)
            flag = struct.unpack('l', read(4))
            if flag[0] == 000000:
                #-find the length of the GRUP record
                seek(match.end())
                length = struct.unpack('l', read(4))
                #-Once the length is found, we read it into the list
                seek(match.start())
                append(GRUPscanner.Scanner(read(length[0])))
            else:
                pass"""
          
        f.close()

        print "sucess!"
        
    def parseMatch(self, seek, read, match):
        #seek = f.seek
        #read = f.read
        append = self.GRUPS.append
        seek(match.end() + 8)
        flag = struct.unpack('l', read(4))
        if flag[0] == 000000:
            #-find the length of the GRUP record
            seek(match.end())
            length = struct.unpack('l', read(4))
            #-Once the length is found, we read it into the list
            seek(match.start())
            append(GRUPscanner.Scanner(read(length[0])))
        else:
            pass
            
if __name__ == '__main__':
    obj = File("H:/Desktop/Living Economy.esp")
    obj.parseRecord()



Sorter.py
import struct
import cStringIO
class BinaryTree():
    def __init__(self, name, data, flag=None, parent=None):
        self.name = name
        self.data = data
        self.left = []
        self.right = []
        self.parent = parent
        self.flag = flag

    def __str__(self):
        if len(self.data) == 16: 
            return "%s: %s" % (self.name, struct.unpack('l4sll', self.data))
        else:
            return "%s" % self.name

    def __len__(self):
        return len(data)
    
    def save(self, File):
        File.write(self.name)
        File.write(self.data)

class GMST(BinaryTree):
    def __init__(self, name, data, parent):
        BinaryTree.__init__(self, name, data, None, parent)
        self.data = cStringIO.StringIO(data)
        self.hedr = self.data.read(16)
        self.edid = self.parse()
        self.fnam = ""

    def __str__(self):
        return "%s" % self.edid[2]
    
    def OnDialog(self):
        self.fnam = self.parse()

    def parse(self):
        title = self.data.read(4)
        length = struct.unpack('H', self.data.read(2))
        info = self.data.read(length[0])
        x = []
        x.append(title)
        x.append(length)
        x.append(info)
        return x

    def save(self, File):
        self.data.seek(0)
        File.write(self.name)
        File.write(self.data.read())

class GLOB(BinaryTree):
    def __init__(self, name, data, parent):
        BinaryTree.__init__(self, name, data, None, parent)
        self.data = cStringIO.StringIO(data)
        self.hedr = self.data.read(16)
        self.edid = self.parse()
        self.fnam = ""
        self.fltv = ""
        
    def __str__(self):
        return "%s" % self.edid[2]
    
    def OnDialog(self):
        self.fnam = self.parse()
        self.fltv = self.parse()

    def parse(self):
        title = self.data.read(4)
        length = struct.unpack('H', self.data.read(2))
        info = self.data.read(length[0])
        x = []
        x.append(title)
        x.append(length)
        x.append(info)
        return x

    def save(self, File):
        self.data.seek(0)
        File.write(self.name)
        File.write(self.data.read())
        

#-------------------------------------------------------------------------------
class Sorter():
    def __init__(self, TokenList):
        self.TreeList = []
        self.tree = None
        self.TokenList = TokenList
        self.GRUPdict = {0: lambda x, y, flag: self.Ctree(x, y, flag),
                    1: lambda x, y, flag: self.Ctree(x, y, flag, self.tree),
                    2: lambda x, y, flag: self.Ctree(x, y, flag, self.tree),
                    3: lambda x, y, flag: self.Ctree(x, y, flag, self.tree),
                    4: lambda x, y, flag: self.Ctree(x, y, flag, self.tree),
                    5: lambda x, y, flag: self.Ctree(x, y, flag, self.tree),
                    6: lambda x, y, flag: self.Ctree(x, y, flag, self.tree),
                    7: lambda x, y, flag: self.Ctree(x, y, flag, self.tree),      
                    8: lambda x, y, flag: self.Ctree(x, y, flag, self.tree),  #TODO: Fix
                    9: lambda x, y, flag: self.Ctree(x, y, flag, self.tree),  #TODO: Fix
                    10: lambda x, y, flag: self.Ctree(x, y, flag, self.tree)} #TODO: Fix
    def run(self):
        self.x = [self.JumpTo(token, data) for token, data in self.TokenList]
        
    def Ctree(self, name, data, flag, parent=None):
        if parent == None:
            newTree = BinaryTree(name, data, flag)
            self.TreeList.append(newTree)
            self.tree = newTree
        else:
            if (flag == 1) | (flag == 2):
                self.tree = self.TreeList[len(self.TreeList) - 1]
                newTree = BinaryTree(name, data, flag, parent)
                self.tree.right.append(newTree)
                self.tree = newTree
            elif flag == 4:
                self.tree = self.TreeList[len(self.TreeList) - 1]
                self.tree = self.tree.right[len(self.tree.right) - 1]
                newTree = BinaryTree(name, data, flag, parent)
                self.tree.right.append(newTree)
                self.tree = newTree
            elif flag == 7:
                newTree = BinaryTree(name, data, flag, parent)
                self.tree.right.append(newTree)
            elif (flag == 8) | (flag == 9) | (flag == 10):
                newTree = BinaryTree(name, data, flag, parent)
                self.tree.right.append(newTree)
            else:
                newTree = BinaryTree(name, data, flag, parent)
                self.tree.right.append(newTree)
                self.tree = newTree
            
    def JumpTo(self, token, data):
        if token == 'GRUP':
            self.GRUPhandler(token, data)
        elif token == 'CELL':
            self.CELLhandler(token, data)
        elif token == 'ROAD':
            self.ROADhandler(token, data)
        elif token == 'WRLD':
            self.WRLDhandler(token, data)
        else:
            self.GenericHandler(token, data)

    def GRUPhandler(self, token, data):
        string = cStringIO.StringIO(data)
        string.seek(8)
        flag = struct.unpack('l', str(string.read(4)))
        self.GRUPdict[flag[0]](token, data, flag[0])

    def CELLhandler(self, token, data):
        if self.tree.flag == 6:
            self.tree = self.tree.parent
        newTree = BinaryTree(token, data)
        self.tree.right.append(newTree)
        
    def ROADhandler(self, token, data):
        newTree = BinaryTree(token, data)
        self.tree.right.append(newTree)

    def WRLDhandler(self, token, data):
        self.tree = self.TreeList[len(self.TreeList) - 1] 
        newTree = BinaryTree(token, data)
        self.tree.right.append(newTree)

    def DIALhandler(self, token, data):
        self.tree = self.TreeList[len(self.TreeList) - 1] 
        newTree = BinaryTree(token, data)
        self.tree.right.append(newTree)

    def GenericHandler(self, token, data):
        if (token == 'REFR') | (token == 'ACHR') | (token == 'ACRE') | (token == 'LAND') | (token == 'PGRD'):
            newTree = BinaryTree(token, data, self.tree)
            self.tree.right[len(self.tree.right) - 1].left.append(newTree)
        elif token == 'GMST':
            newTree = GMST(token, data, self.tree)
            self.tree.left.append(newTree)
        elif token == 'GLOB':
            newTree = GLOB(token, data, self.tree)
            self.tree.left.append(newTree)
        else:
            newTree = BinaryTree(token, data, self.tree)
            self.tree.left.append(newTree)

               
#--------------------------------------------------------------------------------
if __name__ == '__main__':
    TokenList = [['GRUP', '\x15\x01\x00\x00CELL\x00\x00\x00\x00\x1a3\x00\x00'],
                 ['GRUP', '\x01\x01\x00\x00\t\x00\x00\x00\x02\x00\x00\x00\x1a3\x00\x00'],
                 ['GRUP', '\xed\x00\x00\x00\x04\x00\x00\x00\x03\x00\x00\x00\x1a3\x00\x00'],
                 ['CELL', '\x00\x00\x00\x00\x00\x00=\\\x01\x00\x00\x00\x00\x00EDID\x0e\x00AbandonedMine\x00FULL\x0f\x00Abandoned Mine\x00DATA\x01\x00AXCLL$\x00\x16\x13\x10\x00\x00\x00\x00\x00\x19\x14\x0b\x00\x00\x00\x00\x00\x00\x00\xfaD\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x80?\x00\x00\xfaDXCMT\x01\x00\x02GR'],
                 ['GRUP', 'd\x00\x00\x00=\\\x01\x00\x06\x00\x00\x00\x1a3\x00\x00'],
                 ['GRUP', 'P\x00\x00\x00=\\\x01\x00\t\x00\x00\x00\x1a3\x00\x00'],
                 ['REFR', '\x00\x00\x00\x00\x00\x00\xe6\x0c\x00\x01\x1a3\x00\x00NAME\x04\x00zw\x01\x00DATA\x18\x00\x8f;\xffDL\xcf\x00E^\x9a\xfdE\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00']]

    main = Sorter(TokenList)
    main.run()



Now, I'm postive that the slowness is coming from either test.py or GRUPscanner.py, but most likely GRUPscanner.py. This means its related to the cStringIO module, and the many appends I do. Could anybody help me fix this so it doesn't take 5 minutes to load a 10mb, or is my only hope writing a c extension for python. Note, that I've already tried solutions such as Pyrex which have not helped. Also, I have done many goole searches, but haven't found anything helpful.

Share this post


Link to post
Share on other sites
Advertisement
I have, and I determined that the slow down usually comes from test.py or GRUPscanner.py. However, I have no idea how to optimize it, and google was very unhelpful. Which is why I came here, I was wondering if anybody here knew of some good omptimizations in python.

Share this post


Link to post
Share on other sites
The profiler can tell you exactly which functions are slow, not just which files. There's an awful lot of code there, and although I'm actually quite good at optimising Python code, I don't have the time on my hands to read through all of that.

That seek(0) looks pretty pointless though.

Share this post


Link to post
Share on other sites
Python probably isn't the problem here. It looks like most of what you are doing is very high level operations, which python handles very well since its implementations of those operations are fast.

I think the problem is excessive I/O. Read the file once and then work with the string that is returned. For large files, it might be better to read the file and process it incrementally, so that you don't need to hold the entire file in memory. Your code currently reads the entire file, performs pattern matching on it, and then jumps around the file reading off the matches for parsing. It would be better to read the file once and then parse the matches by slicing that string, instead of going back to the file again to read the matches.

Share this post


Link to post
Share on other sites
I found out what the problem was, and now it loads the 10mb file in less than 5 seconds. The problem was that in the GRUPscanner.py module, at the end of the while loop, I was checking the length of the file with len(file.read( )) by declaring a local variable such as len = len(file.read()) and checking against that local variable, I drastically improved the speed.

Share this post


Link to post
Share on other sites
Ah, yes. Re-reading the entire file once for every handful of bytes probably isn't the most efficient method.

Share this post


Link to post
Share on other sites
Sign in to follow this  

  • Advertisement
×

Important Information

By using GameDev.net, you agree to our community Guidelines, Terms of Use, and Privacy Policy.

GameDev.net is your game development community. Create an account for your GameDev Portfolio and participate in the largest developer community in the games industry.

Sign me up!