handle path separators in userpic keywords

[oss/ljdump.git] / ljdump.py

diff --git a/ljdump.py b/ljdump.py
index e7534a91b83f940f69321a0142c743069eb2adb0..d1e92874f7815e076548f24a8e93926bc9542f55 100755 (executable)
--- a/ljdump.py
+++ b/ljdump.py
@@ -2,7 +2,7 @@
 #
 # ljdump.py - livejournal archiver
 # Greg Hewgill <greg@hewgill.com> http://hewgill.com
 #
 # ljdump.py - livejournal archiver
 # Greg Hewgill <greg@hewgill.com> http://hewgill.com

-# Version 1.2
+# Version 1.3.1

 #
 # $Id$
 #
 #
 # $Id$
 #


@@ -26,6 +26,9 @@
 # This program may be run as often as needed to bring the backup copy up
 # to date. Both new and updated items are downloaded.
 #
 # This program may be run as often as needed to bring the backup copy up
 # to date. Both new and updated items are downloaded.
 #

+# The community http://ljdump.livejournal.com has been set up for questions
+# or comments.
+#

 # LICENSE
 #
 # This software is provided 'as-is', without any express or implied
 # LICENSE
 #
 # This software is provided 'as-is', without any express or implied


@@ -44,7 +47,7 @@
 #    misrepresented as being the original software.
 # 3. This notice may not be removed or altered from any source distribution.
 #
 #    misrepresented as being the original software.
 # 3. This notice may not be removed or altered from any source distribution.
 #

-# Copyright (c) 2005-2006 Greg Hewgill
+# Copyright (c) 2005-2009 Greg Hewgill

 
 import codecs, md5, os, pickle, pprint, re, shutil, sys, urllib2, xml.dom.minidom, xmlrpclib
 from xml.sax import saxutils
 
 import codecs, md5, os, pickle, pprint, re, shutil, sys, urllib2, xml.dom.minidom, xmlrpclib
 from xml.sax import saxutils


@@ -106,6 +109,12 @@ def writedump(fn, event):
     dumpelement(f, "event", event)
     f.close()
 
     dumpelement(f, "event", event)
     f.close()
 

+def writelast():
+    f = open("%s/.last" % Username, "w")
+    f.write("%s\n" % lastsync)
+    f.write("%s\n" % lastmaxid)
+    f.close()
+

 def createxml(doc, name, map):
     e = doc.createElement(name)
     for k in map.keys():
 def createxml(doc, name, map):
     e = doc.createElement(name)
     for k in map.keys():


@@ -190,13 +199,18 @@ while True:
                     'selecttype': "one",
                     'itemid': item['item'][2:],
                 }, Password))
                     'selecttype': "one",
                     'itemid': item['item'][2:],
                 }, Password))

-                writedump("%s/%s" % (Username, item['item']), e['events'][0])
-                newentries += 1
+                if e['events']:
+                    writedump("%s/%s" % (Username, item['item']), e['events'][0])
+                    newentries += 1
+                else:
+                    print "Unexpected empty item: %s" % item['item']
+                    errors += 1

             except xmlrpclib.Fault, x:
                 print "Error getting item: %s" % item['item']
                 pprint.pprint(x)
                 errors += 1
         lastsync = item['time']
             except xmlrpclib.Fault, x:
                 print "Error getting item: %s" % item['item']
                 pprint.pprint(x)
                 errors += 1
         lastsync = item['time']

+        writelast()

 
 # The following code doesn't work because the server rejects our repeated calls.
 # http://www.livejournal.com/doc/server/ljp.csp.xml-rpc.getevents.html
 
 # The following code doesn't work because the server rejects our repeated calls.
 # http://www.livejournal.com/doc/server/ljp.csp.xml-rpc.getevents.html


@@ -270,7 +284,14 @@ for p in userpics:
     print >>f, """<userpic keyword="%s" url="%s" />""" % (p, userpics[p])
     pic = urllib2.urlopen(userpics[p])
     ext = MimeExtensions.get(pic.info()["Content-Type"], "")
     print >>f, """<userpic keyword="%s" url="%s" />""" % (p, userpics[p])
     pic = urllib2.urlopen(userpics[p])
     ext = MimeExtensions.get(pic.info()["Content-Type"], "")

-    picf = open("%s/%s%s" % (Username, codecs.utf_8_decode(p)[0], ext), "wb")
+    picfn = re.sub(r"[\/]", "_", p)
+    try:
+        picfn = codecs.utf_8_decode(picfn)[0]
+        picf = open("%s/%s%s" % (Username, picfn, ext), "wb")
+    except:
+        # for installations where the above utf_8_decode doesn't work
+        picfn = "".join([ord(x) < 128 and x or "?" for x in picfn])
+        picf = open("%s/%s%s" % (Username, picfn, ext), "wb")

     shutil.copyfileobj(pic, picf)
     pic.close()
     picf.close()
     shutil.copyfileobj(pic, picf)
     pic.close()
     picf.close()


@@ -320,10 +341,7 @@ while True:
 
 lastmaxid = maxid
 
 
 lastmaxid = maxid
 

-f = open("%s/.last" % Username, "w")
-f.write("%s\n" % lastsync)
-f.write("%s\n" % lastmaxid)
-f.close()
+writelast()

 
 if origlastsync:
     print "%d new entries, %d new comments (since %s)" % (newentries, newcomments, origlastsync)
 
 if origlastsync:
     print "%d new entries, %d new comments (since %s)" % (newentries, newcomments, origlastsync)