1 # TODO:
    2 #
    3 # - Find out how to binary diff the ape files so that I can work out why they
    4 #   differ (usually recompiled versions are bigger) when I recompile them.
    5 #
    6 #   Need to write a script that goes through all the ape files and decompiles
    7 #   them and recompiles them, noting the errors both in the decompilation
    8 #   process and in the recompilation process.  Perhaps spit out a test report
    9 #   in HTML which links to the output from both parts.
   10 #
   11 
   12 import sys
   13 import struct
   14 from StringIO import StringIO
   15 
   16 DEBUG = 0
   17 
   18 def Binary(x1, bits=32):
   19     """ x1: The number to convert.
   20         bits: How many bits make up the number.
   21 
   22         Purpose: Create a string representation of the numbers binary form.
   23 
   24         This takes a number and returns a readable string that represents
   25         each bit in the number with a '0' or '1' character respectively.
   26         e.g. Binary(12, 4)
   27              "1100"
   28     """
   29     s = ""
   30     x = x1
   31     for i in range(bits):
   32         s = ['0','1'][x & 1] + s
   33         x = x >> 1
   34     return s
   35 
   36 def CountBits(x1):
   37     """ x1: The long sized value to count.
   38 
   39         Purpose: Count the number of leading 0 bits in the value.
   40 
   41         This has a one bit mask which it shifts from the high end to the
   42         low end of the long until the whole long is covered or a 1 bit is
   43         found.
   44     """
   45     cnt = 0
   46     mask = 0x8000000000000000L
   47     while mask and not (x1 & mask):
   48         cnt += 1
   49         mask = mask >> 1
   50     return cnt
   51 
   52 def IntegerToLabel(v):
   53     """ v: The integer which is an APE label.
   54 
   55         Purpose: Take the compiled label form and convert it to the APE code label form.
   56     """
   57     if v == 0:
   58         return "0:0"
   59     s = str(v)
   60     s1, s2 = s[:-4], int(s[-4:])
   61     return "%s:%s" % (s1, s2)
   62 
   63 def ReadByte(f):
   64     return f.read(1)
   65 
   66 def ReadWord(f):
   67     return struct.unpack("<H", f.read(2))[0]
   68 
   69 def ReadInteger(f):
   70     return struct.unpack("<I", f.read(4))[0]
   71 
   72 def ReadLong(f):
   73     return struct.unpack("<q", f.read(8))[0]
   74 
   75 def ReadFloat(f):
   76     return struct.unpack("<f", f.read(4))[0]
   77 
   78 def ReadDouble(f):
   79     return struct.unpack("<d", f.read(8))[0]
   80 
   81 def ReadString(f, bytes=None):
   82     if bytes is None:
   83         bytes = ReadInteger(f)
   84     if bytes > 1000:
   85         raise RuntimeError("string length too large", bytes)
   86     s = f.read(bytes-1)
   87     f.read(1)
   88     return s
   89 
   90 def ReadString2(f):
   91     s = ReadString(f)
   92     if s[-1] == '\n':
   93         s = s[:-1]
   94     return s
   95 
   96 def ReadQuotedString(f, bytes=None):
   97     s = ReadString(f, bytes)
   98     s = s.replace("\\", "\\\\")
   99     s = s.replace("\n", "\\n")
  100     s = s.replace("\"", "\\\"")
  101     return '"'+ s +'"'
  102 
  103 def DumpString(s):
  104     if not len(s):
  105         return
  106     print "  DUMP %d\n" % len(s)
  107     print "  ",
  108     for c in s:
  109         print "%02x " % ord(c),
  110     print
  111 
  112 def ReadBytes(f, n):
  113     return f.read(n)
  114 
  115 def ReadAndDump(f, out=None, cnt=None):
  116     if cnt is None:
  117         s = f.read()
  118     else:
  119         s = f.read(cnt)
  120     DumpString(s)
  121     if cnt is None:
  122         if out is not None:
  123             print out.getvalue()
  124         raise RuntimeError("ReadAndDump", out)
  125 
  126 operators = {
  127     0x01: "||",
  128     0x02: "&&",
  129     0x03: "^^",
  130     0x04: ">",
  131     0x05: "<",
  132     0x06: ">=",
  133     0x07: "<=",
  134     0x08: "==",
  135     0x09: "+",
  136     0x0a: "-",
  137     0x0b: "*",
  138     0x0c: "/",
  139     0x0d: "!=",
  140 }
  141 
  142 class Expression:
  143     operator = None
  144     value1 = None
  145     op1 = None
  146     value2 = None
  147     op2 = None
  148     def __init__(self, operand, op):
  149         self.operand = operand
  150         self.op = op
  151 
  152         self.operator = operators.get(self.operand, None)
  153         if self.operator is None:
  154             raise RuntimeError("Bad operand", operand, op)
  155 
  156     def Read1(self, f, op):
  157         if self.op in (0x00, 0x08, 0x0a):
  158             self.value1 = "("+ str(ReadExpression(f)) +")"
  159         elif self.op in (0x04, 0x0c, 0x0e):
  160             self.value1 = ReadFloat(f)
  161         elif self.op in (0x05, 0x0d, 0x0f, 0x31):
  162             self.value1 = ReadString(f)
  163             if self.value1 == "":
  164                 print "**UNEXPECTED INPUT, 1", hex(self.op)
  165         elif self.op in (0x30, 0x32, 0x33):
  166             self.value1 = ReadQuotedString(f)
  167         else:
  168             print "Bad operator1", hex(op), repr(self)
  169             ReadAndDump(f)
  170         self.op1 = op
  171 
  172     def Read2(self, f, op):
  173         if self.op in (0x00, 0x04, 0x05):
  174             self.value2 = "("+ str(ReadExpression(f)) +")"
  175         elif self.op in (0x08, 0x0c, 0x0d):
  176             self.value2 = ReadFloat(f)
  177         elif self.op in (0x0a, 0x0e, 0x0f):
  178             self.value2 = ReadString(f)
  179             if self.value2 == "":
  180                 print "**UNEXPECTED INPUT, 2", hex(self.op)
  181         elif self.op in (0x30, 0x31, 0x32, 0x33):
  182             self.value2 = ReadQuotedString(f)
  183         else:
  184             print "Bad operator2", hex(op), repr(self)
  185             ReadAndDump(f)
  186         self.op2 = op
  187 
  188     def __float__(self):
  189         if self.operator == "+" and self.value1 == 0.0:
  190             return self.value2
  191         raise
  192 
  193     def __str__(self):
  194         if self.operator == "+" and self.value1 == 0.0:
  195             return str(self.value2)
  196         return "%s %s %s" % (self.value1, self.operator, self.value2)
  197 
  198     def __repr__(self):
  199         return "<%s operator=\"%s\" op=%s op1=%s value1=%s op2=%s value2=%s>" % (
  200             self.__class__.__name__,
  201             self.operator,
  202             self.op and "$%02x" % self.op,
  203             self.op1 and "$%02x" % self.op1,
  204             self.value1,
  205             self.op2 and "$%02x" % self.op2, # Binary(self.op2),
  206             self.value2)
  207 
  208 
  209 def ReadExpression(f):
  210     op1 = ord(ReadByte(f))
  211     op2 = ord(ReadByte(f))
  212     expr = Expression(op1, op2)
  213 
  214     v2 = ReadLong(f)
  215     expr.Read1(f, v2)
  216 
  217     v3 = ReadLong(f)
  218     expr.Read2(f, v3)
  219 
  220     return expr
  221 
  222 class Formatting:
  223     def __init__(self):
  224         self.members = [ '' ]
  225     def Add(self, x):
  226         self.members.append(x)
  227     def __str__(self):
  228         if len(self.members) == 1:
  229             return ""
  230         return ", ".join([ str(x) for x in self.members ])
  231     def __len__(self):
  232         return len(self.members) - 1
  233 
  234 def ReadFormatting(f, ret=None):
  235     if ret is None:
  236         ret = Formatting()
  237     b1 = ord(ReadByte(f))
  238     b2 = ord(ReadByte(f))
  239     if b1 == 0xff and b2 == 0xff:
  240         return ret
  241     if b1 == 0:
  242         done = 1
  243         if b2 == 0x05:
  244             ret.Add(ReadString(f))
  245         elif b2 == 0x04:
  246             ret.Add(ReadFloat(f))
  247         elif b2 == 0x10:
  248             ret.Add(ReadQuotedString(f))
  249         elif b2 == 0x11:
  250             ret.Add(ReadString(f))
  251         else:
  252             done = 0
  253         if done:
  254             return ReadFormatting(f, ret)
  255 
  256     print "** Error in ReadFormatting"
  257     print hex(b1), Binary(b1, 8)
  258     print hex(b2), Binary(b2, 8)
  259     ReadAndDump(f)
  260 
  261 def ReadExpressionSegment(f):
  262     v1, v2 = None, None
  263     y = ReadLong(f)
  264     if y == 1:
  265         v1 = ReadExpression(f)
  266         x = ReadLong(f)
  267         if x == 1:
  268             v2 = ReadExpression(f)
  269         elif x != 0:
  270             print "ReadExpressionSegment.x", x
  271             ReadAndDump(f)
  272     elif y != 0:
  273         print "ReadExpressionSegment.y", y
  274         ReadAndDump(f)
  275     cc = ReadLong(f)
  276     return v1, v2, cc
  277 
  278 def AssumeIgnoredExpressionSegment(f):
  279     v1, v2, cc = ReadExpressionSegment(f)
  280     if v1 is not None or v2 is not None:
  281         print "AssumeIgnoredExpressionSegment", v1, v2
  282         ReadAndDump(f)
  283     return cc
  284 
  285 lastCC = 0
  286 
  287 class NestedCommands:
  288     name = None
  289     condition = None
  290     exitCount = 0
  291 
  292     def __init__(self):
  293         self.values = []
  294 
  295     def Add(self, cmd):
  296         self.values.append(cmd)
  297 
  298     def Dump(self, indent=0):
  299         sys.stderr.write("  " * indent)
  300         sys.stderr.write("[*] NestedCommands %s %s, length %d\n" % (self.name, self.condition, len(self.values)))
  301         for each in self.values:
  302             sys.stderr.write("  " * indent)
  303             if type(each) is str:
  304                 sys.stderr.write("%s\n" % each)
  305             else:
  306                 each.Dump(indent+1)
  307             sys.stderr.write("\n")
  308         sys.stderr.write("\n")
  309 
  310 def ReadCommands(f, cmds, refs):
  311     global lastCC
  312     ret = 1
  313     while ret:
  314         ret, lastCC = ReadCommand(f, cmds, refs)
  315 
  316 
  317 def ReadNestedCommands(f, cc):
  318     # When more bits are added, its a new statement of deeper nestedness.
  319     # When bits are removed, its dropping back levels of nestedness.
  320     shiftMaskIn    = 0xfffffffffffffffcL
  321     shiftMaskInNew = 0x0000000000000003L
  322 
  323     firstCC = cc
  324     cnt0 = CountBits(firstCC)
  325 
  326     cmds = NestedCommands()
  327     try:
  328         while 1:
  329             lastCC = cc
  330             ret, cc = ReadCommand(f, cmds)
  331             if ret == 0:
  332                 print "Error: if ret/cc2 "+ Binary(cc)
  333                 print cmds
  334                 ReadAndDump(f)
  335             if cc is None:
  336                 raise RuntimeError("Expected condition code", cmds)
  337 
  338             lastCC1 = lastCC << 2
  339             if cc & shiftMaskIn != lastCC1:
  340                 cnt1 = CountBits(lastCC)
  341                 cnt2 = CountBits(cc)
  342                 if cnt1 < cnt2:
  343                     if cc & shiftMaskInNew == 2:
  344                         cmds.exitCount = 0
  345                         while cnt1 < cnt2:
  346                             # Get the next sequence and move the lastCC to account for it.
  347                             v = lastCC & shiftMaskInNew
  348                             lastCC = lastCC >> 2
  349                             cnt1 += 2
  350                             if v == 2:
  351                                 cmds.exitCount += 1
  352                         if cmds.exitCount > 0:
  353                             cmds.exitCount -= 1
  354                             break
  355                         cmds.Add(None)
  356                     else:
  357                         # Last statement in a nesting.
  358                         break
  359                 elif cnt1 == cnt2 and lastCC & shiftMaskInNew in (1, 2) and cc & shiftMaskInNew == 3:
  360                     # This was the only clause following an else.  Drop out to give the next clause to the container.
  361                     break
  362                 elif cnt1 == cnt2 and lastCC & shiftMaskInNew == 1 and cc & shiftMaskInNew == 2:
  363                     cmds.Add(None)
  364                 else:
  365                     raise RuntimeError("Unexpected movement", Binary(lastCC, 64), cnt1, Binary(cc, 64), cnt2, cmds)
  366             elif cc & shiftMaskInNew != 3:
  367                 raise RuntimeError("Unexpected flags", cc & shiftMaskInNew)
  368     except KeyboardInterrupt:
  369         print f.tell()
  370         print cmds
  371         raise
  372 
  373     return cmds, cc
  374 
  375 stringCommands = {
  376     4: ("goto", 0),
  377     5: ("gosub", 0),
  378     6: ("console", 1),
  379     7: ("echo", 1),
  380     8: ("target", 0),
  381     9: ("pathtarget", 0),
  382     10: ("extern", 0),
  383     12: ("playambient", 0),
  384     13: ("loopambient", 0),
  385     14: ("stopambient", 0),
  386     15: ("playscene", 0),
  387     16: ("loopscene", 0),
  388     17: ("stopscene", 0),
  389     18: ("chainscripts", 0),
  390     19: ("closewindow", 0),
  391     20: ("loadape", 1),
  392 }
  393 
  394 def ReadCommand(f, cmds, refs=None):
  395     command = ord(f.read(1))
  396     cc = None
  397     if command == 0:
  398         v = ReadInteger(f)
  399         fmt = ReadFormatting(f)
  400         v1, v2, cc = ReadExpressionSegment(f)
  401     elif command in (1, 11):
  402         v = ReadInteger(f)
  403         if v != 0:
  404             print "Error: if value "+ hex(v)
  405             ReadAndDump(f)
  406         fmt = ReadFormatting(f)
  407         if len(fmt):
  408             print "Error: if formatting "+ str(fmt)
  409         v1, v2, cc = ReadExpressionSegment(f)
  410         if cc & 3 != 1:
  411             print "Error: if cc "+ Binary(cc)
  412             ReadAndDump(f)
  413 
  414         entry, cc = ReadNestedCommands(f, cc)
  415         entry.name = [ "while", "if" ][command == 1]
  416         entry.condition = v1
  417 
  418         cmds.Add(entry)
  419     elif command == 2:
  420         variable = ReadString(f)
  421         fmt = ReadFormatting(f)
  422         v1, v2, cc = ReadExpressionSegment(f)
  423         if v1 is not None:
  424             cmds.Add("set %s = %s%s\n" % (variable, v1, str(fmt)))
  425             if v2 is not None:
  426                 cmds.Add("// ?set? %s\n" % v2)
  427         else:
  428             cmds.Add("unset %s\n" % variable)
  429     elif command == 3:
  430         variable = ReadString(f)
  431         fmt = ReadFormatting(f)
  432         v1, v2, cc = ReadExpressionSegment(f)
  433         if v1 is None and v2 is None:
  434             cmds.Add("%s%s\n" % (variable, str(fmt)))
  435         else:
  436             print "** Unexpected v1 or v2 in string var set", v1, v2
  437             ReadAndDump(f)
  438     elif stringCommands.has_key(command):
  439         if stringCommands[command][1]:
  440             txt = ReadQuotedString(f)
  441         else:
  442             txt = ReadString(f)
  443         fmt = ReadFormatting(f)
  444         cmds.Add("%s %s%s\n" % (stringCommands[command][0], txt, fmt))
  445         cc = AssumeIgnoredExpressionSegment(f)
  446     elif command == 49:
  447         label = IntegerToLabel(ReadInteger(f))
  448         cmds.Add("startswitch %s" % label)
  449         refs.append(label)
  450     elif command == 50:
  451         label = IntegerToLabel(ReadInteger(f))
  452         cmds.Add("thinkswitch %s" % label)
  453         refs.append(label)
  454     elif command == 51:
  455         label = IntegerToLabel(ReadInteger(f))
  456         cmds.Add("finishswitch %s" % label)
  457         refs.append(label)
  458     elif command == 65:
  459         cmds.Add("startconsole \"%s\"\n" % ReadString2(f))
  460     elif command == 66:
  461         conditions = []
  462         v1 = ReadLong(f)
  463         while v1 == 1:
  464             conditions.append(ReadExpression(f))
  465             v1 = ReadLong(f)
  466         s = ReadQuotedString(f)
  467         fmt = ReadFormatting(f)
  468         if len(conditions) == 1:
  469             cmds.Add("if (%s)\n  " % conditions[0])
  470         elif len(conditions):
  471             raise RuntimeError("body, too many conditions", conditions)
  472         cmds.Add("body %s%s\n" % (s, fmt))
  473     elif command == 67:
  474         conditions = []
  475         v1 = ReadLong(f)
  476         while v1 == 1:
  477             conditions.append(ReadExpression(f))
  478             v1 = ReadLong(f)
  479         text = ReadQuotedString(f)
  480         fmt = ReadFormatting(f)
  481         if len(conditions) == 1:
  482             cmds.Add("if (%s)\n  " % conditions[0])
  483         elif len(conditions):
  484             raise RuntimeError("body, too many conditions", conditions)
  485         cmds.Add("choice %s %s%s\n" % (text, IntegerToLabel(ReadInteger(f)), fmt))
  486     elif command == 68:
  487         out = StringIO()
  488         out.write("background")
  489         for i in range(1, 5):
  490             out.write(" color%d=%02x%02x%02x%02x" % (i, ord(ReadByte(f)), ord(ReadByte(f)), ord(ReadByte(f)), ord(ReadByte(f))))
  491         out.write("\n")
  492         cmds.Add(out.getvalue())
  493     elif command == 70:
  494         cmds.Add("font \"%s\"\n" % ReadString(f))
  495     elif command == 71:
  496         tokens = (("xpos", 0), ("ypos", 0), ("width", 1), ("height", 1))
  497         tokenValues = {}
  498         for token, moreIfSet in tokens:
  499             v = ReadLong(f)
  500             if v == 1:
  501                 tokenValues[token] = ReadExpression(f)
  502                 cmds.Add("%s %s\n" % (token, tokenValues[token]))
  503 
  504                 v = ReadLong(f)
  505                 if v == 1:
  506                     cmds.Add("%s-x %s\n" % (token, ReadExpression(f)))
  507     elif command == 72:
  508         v = ReadLong(f)
  509         if v != 0:
  510             print "** error in subwindow", v
  511             ReadAndDump(f)
  512         cmds.Add("subwindow %s\n" % IntegerToLabel(ReadInteger(f)))
  513     elif command == 73:
  514         conditions = []
  515         v1 = ReadLong(f)
  516         while v1 == 1:
  517             conditions.append(ReadExpression(f))
  518             v1 = ReadLong(f)
  519 
  520         fileName = ReadString(f)
  521         tokens = (("xpos", 0), ("ypos", 0), ("width", 1), ("height", 1))
  522         tokenValues = {}
  523         for token, moreIfSet in tokens:
  524             v = ReadLong(f)
  525             if v == 1:
  526                 tokenValues[token] = ReadExpression(f)
  527                 v = ReadLong(f)
  528                 if v == 1:
  529                     tokenValues[token +"2"] = ReadExpression(f)
  530         flags = ReadInteger(f)
  531         l = []
  532         if flags & 1:
  533             l.append("stretch")
  534         if flags & 2:
  535             l.append("tile")
  536         if flags & 4:
  537             l.append("solid")
  538         flagString = ''.join([ ', "'+ s+ '"' for s in l ])
  539         if len(conditions) == 1:
  540             cmds.Add("if (%s)\n  " % conditions[0])
  541         elif len(conditions) > 1:
  542             raise RuntimeError("More conditions than expected", conditions)
  543         if not (tokenValues.has_key("width") or tokenValues.has_key("height")):
  544             cmds.Add("image \"%s\" %s,%s%s\n" % (fileName, tokenValues['xpos'], tokenValues['ypos'], flagString))
  545         else:
  546             cmds.Add("image \"%s\" %s,%s, %s,%s%s\n" % (fileName, tokenValues['xpos'],tokenValues['ypos'],tokenValues['width'],tokenValues['height'], flagString))
  547     elif command ==