From: greg <greg@7994a137-d0a5-da11-ade6-0050bffea3d9>
Date: Tue, 10 Jan 2006 04:11:07 +0000 (+0000)
Subject: add comment functionality
X-Git-Tag: ljdump-1.3.2~17
X-Git-Url: http://wagner.pp.ru/gitweb/?a=commitdiff_plain;h=74970e50b7ec3994f4408475134a29d496ddb510;p=oss%2Fljdump.git

add comment functionality


git-svn-id: file:///home/svn/ljdump/trunk@15 7994a137-d0a5-da11-ade6-0050bffea3d9
---

diff --git a/ljdump.py b/ljdump.py
index e642974..751dfff 100755
--- a/ljdump.py
+++ b/ljdump.py
@@ -2,7 +2,7 @@
 #
 # ljdump.py - livejournal archiver
 # Greg Hewgill <greg@hewgill.com> http://hewgill.com
-# Version 1.0.3
+# Version 1.1
 #
 # $Id$
 #
@@ -44,17 +44,43 @@
 #    misrepresented as being the original software.
 # 3. This notice may not be removed or altered from any source distribution.
 #
-# Copyright (c) 2005 Greg Hewgill
+# Copyright (c) 2005-2006 Greg Hewgill
 
-import codecs, md5, os, pprint, sys, xml.dom.minidom, xmlrpclib
+import codecs, md5, os, pickle, pprint, sys, urllib2, xml.dom.minidom, xmlrpclib
 from xml.sax import saxutils
 
+def calcchallenge(challenge, password):
+    return md5.new(challenge+md5.new(password).hexdigest()).hexdigest()
+
+def flatresponse(response):
+    r = {}
+    while True:
+        name = response.readline()
+        if len(name) == 0:
+            break
+        if name[-1] == '\n':
+            name = name[:len(name)-1]
+        value = response.readline()
+        if value[-1] == '\n':
+            value = value[:len(value)-1]
+        r[name] = value
+    return r
+
+def getljsession(username, password):
+    r = urllib2.urlopen("http://livejournal.com/interface/flat", "mode=getchallenge")
+    response = flatresponse(r)
+    r.close()
+    r = urllib2.urlopen("http://livejournal.com/interface/flat", "mode=sessiongenerate&user=%s&auth_method=challenge&auth_challenge=%s&auth_response=%s" % (username, response['challenge'], calcchallenge(response['challenge'], password)))
+    response = flatresponse(r)
+    r.close()
+    return response['ljsession']
+
 def dochallenge(params, password):
     challenge = server.LJ.XMLRPC.getchallenge()
     params.update({
         'auth_method': "challenge",
         'auth_challenge': challenge['challenge'],
-        'auth_response': md5.new(challenge['challenge']+md5.new(password).hexdigest()).hexdigest()
+        'auth_response': calcchallenge(challenge['challenge'], password)
     })
     return params
 
@@ -74,6 +100,19 @@ def writedump(fn, event):
     dumpelement(f, "event", event)
     f.close()
 
+def createxml(doc, name, map):
+    e = doc.createElement(name)
+    for k in map.keys():
+        me = doc.createElement(k)
+        me.appendChild(doc.createTextNode(map[k]))
+        e.appendChild(me)
+    return e
+
+def gettext(e):
+    if len(e) == 0:
+        return ""
+    return e[0].firstChild.nodeValue
+
 config = xml.dom.minidom.parse("ljdump.config")
 Server = config.documentElement.getElementsByTagName("server")[0].childNodes[0].data
 Username = config.documentElement.getElementsByTagName("username")[0].childNodes[0].data
@@ -86,27 +125,38 @@ try:
 except:
     pass
 
+ljsession = getljsession(Username, Password)
+
 server = xmlrpclib.ServerProxy(Server)
 
-new = 0
+newentries = 0
+newcomments = 0
 errors = 0
 
-last = ""
+lastsync = ""
+lastmaxid = 0
 try:
     f = open("%s/.last" % Username, "r")
-    last = f.readline()
-    if last[-1] == '\n':
-        last = last[:len(last)-1]
+    lastsync = f.readline()
+    if lastsync[-1] == '\n':
+        lastsync = lastsync[:len(lastsync)-1]
+    lastmaxid = f.readline()
+    if len(lastmaxid) > 0 and lastmaxid[-1] == '\n':
+        lastmaxid = lastmaxid[:len(lastmaxid)-1]
+    if lastmaxid == "":
+        lastmaxid = 0
+    else:
+        lastmaxid = int(lastmaxid)
     f.close()
 except:
     pass
-origlast = last
+origlastsync = lastsync
 
 while True:
     r = server.LJ.XMLRPC.syncitems(dochallenge({
         'username': Username,
         'ver': 1,
-        'lastsync': last,
+        'lastsync': lastsync,
     }, Password))
     #pprint.pprint(r)
     if len(r['syncitems']) == 0:
@@ -122,18 +172,128 @@ while True:
                     'itemid': item['item'][2:],
                 }, Password))
                 writedump("%s/%s" % (Username, item['item']), e['events'][0])
-                new += 1
+                newentries += 1
             except xmlrpclib.Fault, x:
                 print "Error getting item: %s" % item['item']
                 pprint.pprint(x)
                 errors += 1
-        last = item['time']
+        lastsync = item['time']
+
+# The following code doesn't work because the server rejects our repeated calls.
+# http://www.livejournal.com/doc/server/ljp.csp.xml-rpc.getevents.html
+# contains the statement "You should use the syncitems selecttype in
+# conjuntions [sic] with the syncitems protocol mode", but provides
+# no other explanation about how these two function calls should
+# interact. Therefore we just do the above slow one-at-a-time method.
+
+#while True:
+#    r = server.LJ.XMLRPC.getevents(dochallenge({
+#        'username': Username,
+#        'ver': 1,
+#        'selecttype': "syncitems",
+#        'lastsync': lastsync,
+#    }, Password))
+#    pprint.pprint(r)
+#    if len(r['events']) == 0:
+#        break
+#    for item in r['events']:
+#        writedump("%s/L-%d" % (Username, item['itemid']), item)
+#        newentries += 1
+#        lastsync = item['eventtime']
+
+print "Fetching journal comments for: %s" % Username
+
+try:
+    f = open("%s/comment.meta" % Username)
+    metacache = pickle.load(f)
+    f.close()
+except:
+    metacache = {}
+
+try:
+    f = open("%s/user.map" % Username)
+    usermap = pickle.load(f)
+    f.close()
+except:
+    usermap = {}
+
+maxid = lastmaxid
+while True:
+    r = urllib2.urlopen(urllib2.Request("http://livejournal.com/export_comments.bml?get=comment_meta&startid=%d" % (maxid+1), headers = {'Cookie': "ljsession="+ljsession}))
+    meta = xml.dom.minidom.parse(r)
+    r.close()
+    for c in meta.getElementsByTagName("comment"):
+        id = int(c.getAttribute("id"))
+        metacache[id] = {
+            'posterid': c.getAttribute("posterid"),
+            'state': c.getAttribute("state"),
+        }
+        if id > maxid:
+            maxid = id
+    for u in meta.getElementsByTagName("usermap"):
+        usermap[u.getAttribute("id")] = u.getAttribute("user")
+    if maxid >= int(meta.getElementsByTagName("maxid")[0].firstChild.nodeValue):
+        break
+
+f = open("%s/comment.meta" % Username, "w")
+pickle.dump(metacache, f)
+f.close()
+
+f = open("%s/user.map" % Username, "w")
+pickle.dump(usermap, f)
+f.close()
+
+newmaxid = maxid
+maxid = lastmaxid
+while True:
+    r = urllib2.urlopen(urllib2.Request("http://livejournal.com/export_comments.bml?get=comment_body&startid=%d" % (maxid+1), headers = {'Cookie': "ljsession="+ljsession}))
+    meta = xml.dom.minidom.parse(r)
+    r.close()
+    for c in meta.getElementsByTagName("comment"):
+        id = int(c.getAttribute("id"))
+        jitemid = c.getAttribute("jitemid")
+        comment = {
+            'id': str(id),
+            'parentid': c.getAttribute("parentid"),
+            'subject': gettext(c.getElementsByTagName("subject")),
+            'date': gettext(c.getElementsByTagName("date")),
+            'body': gettext(c.getElementsByTagName("body")),
+            'state': metacache[id]['state'],
+        }
+        if usermap.has_key(c.getAttribute("posterid")):
+            comment["user"] = usermap[c.getAttribute("posterid")]
+        try:
+            entry = xml.dom.minidom.parse("%s/C-%s" % (Username, jitemid))
+        except:
+            entry = xml.dom.minidom.getDOMImplementation().createDocument(None, "comments", None)
+        found = False
+        for d in entry.getElementsByTagName("comment"):
+            if int(d.getElementsByTagName("id")[0].firstChild.nodeValue) == id:
+                found = True
+                break
+        if found:
+            print "Warning: downloaded duplicate comment id %d in jitemid %s" % (id, jitemid)
+        else:
+            entry.documentElement.appendChild(createxml(entry, "comment", comment))
+            f = codecs.open("%s/C-%s" % (Username, jitemid), "w", "UTF-8")
+            entry.writexml(f)
+            f.close()
+            newcomments += 1
+        if id > maxid:
+            maxid = id
+    if maxid >= newmaxid:
+        break
+
+lastmaxid = maxid
+
 f = open("%s/.last" % Username, "w")
-f.write("%s\n" % last)
+f.write("%s\n" % lastsync)
+f.write("%s\n" % lastmaxid)
 f.close()
-if origlast:
-    print "%d new entries (since %s)" % (new, origlast)
+
+if origlastsync:
+    print "%d new entries, %d new comments (since %s)" % (newentries, newcomments, origlastsync)
 else:
-    print "%d new entries" % new
+    print "%d new entries, %d new comments" % (newentries, newcomments)
 if errors > 0:
     print "%d errors" % errors