X-Git-Url: https://wagner.pp.ru/gitweb/?a=blobdiff_plain;f=convertdump.py;h=0f59e8282d6dc9e2e9fa481a1312201b6468c2bf;hb=d45d3ae7e90b1e0f3f2d2f93fde04b0312027058;hp=a81b596a22b505f78f2e2e5db0fe28a36e444225;hpb=7fce5f0f25c1887325f41777daae12fcc078a99d;p=oss%2Fljdump.git diff --git a/convertdump.py b/convertdump.py index a81b596..0f59e82 100755 --- a/convertdump.py +++ b/convertdump.py @@ -1,9 +1,36 @@ #!/usr/bin/python +# Copyright 2009, Sean M. Graham (www.sean-graham.com) +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# - Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# - Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +# WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +# EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, +# OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, +# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + import xml.dom.minidom import os import codecs import sys +import getopt +import re from time import strptime, strftime @@ -40,13 +67,12 @@ def appendTextNode(doc, parent, nodename, value): parent.appendChild(element) -def addEntryForId(outDoc, element, username, id): +def addEntryForId(outDoc, element, username, id, includeSecure): entryFile = open("%s/L-%s" % (username,id), "r") inDoc = xml.dom.minidom.parse(entryFile) # Create an entry element entry = outDoc.createElement("entry") - element.appendChild(entry) # Create an itemid element appendTextNode(outDoc, entry, "itemid", getNodeText(inDoc,"itemid")) @@ -60,17 +86,32 @@ def addEntryForId(outDoc, element, username, id): # Create an event node (special case because for some reason there are two # 'event' elements in the pydump output, which is probably LJ's fault) event = inDoc.getElementsByTagName("event")[0] - appendTextNode(outDoc, entry, "event", getNodeText(event, "event")) + eventText = getNodeText(event, "event") + + appendTextNode(outDoc, entry, "event", replaceLJTags(eventText)) + + security = getNodeText(inDoc, "security") + + if(security != ""): + # don't append this entry unless the user provided the argument + if(includeSecure == False): + print("omitting secure entry: L-%s" % id) + return + else: + if(security == "usemask"): + print("including allowmask entry: L-%s" % id) + + # Create an allowmask element + maskText = getNodeText(inDoc, "allowmask") - # Create an allowmask element (doesn't exist in pydump output if public) - maskText = getNodeText(inDoc, "allowmask") + if(maskText != ""): + appendTextNode(outDoc, entry, "allowmask", maskText) + else: + appendTextNode(outDoc, entry, "allowmask", "0") + else: + print("including private entry: L-%s" % id) - # XXXSMG: consult L-1411 and L-976 for examples of security and - # allowmask use - if(maskText != ""): - appendTextNode(outDoc, entry, "allowmask", maskText) - else: - appendTextNode(outDoc, entry, "allowmask", "0") + appendTextNode(outDoc, entry, "security", security) # Create a taglist element appendTextNode(outDoc, entry, "taglist", getNodeText(inDoc, "taglist")) @@ -79,6 +120,8 @@ def addEntryForId(outDoc, element, username, id): # with it addCommentsForId(outDoc, entry, username, id) + element.appendChild(entry) + def addCommentsForId(outDoc, entry, username, id): try: commentFile = open("%s/C-%s" % (username,id), "r") @@ -112,8 +155,8 @@ def addCommentsForId(outDoc, entry, username, id): getNodeText(comment, "subject")) # Create an event element - appendTextNode(outDoc, outComment, "event", - getNodeText(comment, "body")) + bodyText = getNodeText(comment, "body") + appendTextNode(outDoc, outComment, "event", replaceLJTags(bodyText)) # Create the author element author = outDoc.createElement("author") @@ -132,17 +175,94 @@ def addCommentsForId(outDoc, entry, username, id): if(parentId != ""): appendTextNode(outDoc, outComment, "parent_itemid", parentId) + +# regular expressions used in replaceLJTags() +# (global for later reuse - suggestion by jparise) + +userRE = re.compile('', re.IGNORECASE) +commRE = re.compile('', re.IGNORECASE) +namedCutRE = re.compile('', + re.IGNORECASE|re.DOTALL) +cutRE = re.compile('', re.IGNORECASE) +cutRE = re.compile('', re.IGNORECASE) +embedRE = re.compile('', re.IGNORECASE) + +def replaceLJTags(entry): + rv = entry + + # replace lj user tags + rv = re.sub(userRE, '\\1', rv) + + # replace lj comm tags + rv = re.sub(commRE, '\\1', rv) + + # replace lj-cut tags + rv = re.sub(namedCutRE, '', rv) + rv = re.sub(cutRE, '', rv) + rv = re.sub(cutRE, '', rv) + + # replace lj-embed tags + # this doesn't actually work. LJ doesn't include the embedded content + # when ljdump calls 'getevents', but instead includes an lj-embed tag + # with an id and nothing else. + #rv = re.sub(embedRE, '', rv) + + return rv + + +def usage(): + print( "Usage: convertdump.py [arguments]" ) + print( """ +This will convert a pydump archive into something compatible with the +WordPress LiveJournal importer. This is the same format used by the Windows +ljArchive exporter. + +Arguments: + -u --user username of archive to process [required] + -l --limit limit the number of entries in each xml file (default 250) + -i --insecure include private and protected entries in the output + -h --help show this help page + +Example: + ./convertdump.py --user stevemartin --limit 200 --insecure +""") + + def main(argv): username = "" entryLimit = 250 - + includeSecure = False; - if( len(argv) != 2 ): - print( "Usage: convertdump.py " ) - return - else: - username = argv[0] - entryLimit = int(argv[1]) + if( len(argv) == 0 ): + usage() + sys.exit(2) + + try: + opts, args = getopt.getopt(sys.argv[1:], "hu:l:i", ["help", + "user=", + "limit=", + "insecure"]) + except getopt.GetoptError, err: + # print help information and exit: + print str(err) # will print something like "option -a not recognized" + usage() + sys.exit(2) + + for o, a in opts: + if o == "-v": + verbose = True + elif o in ("-u", "--user"): + username = a + elif o in ("-l", "--limit"): + entryLimit = int(a) + elif o in ("-i", "--insecure"): + print( "Warning: Including secure entries in XML output" ) + includeSecure = True + elif o in ("-h", "--help"): + usage() + sys.exit() + else: + assert False, "unhandled option" userDir = os.listdir(username) @@ -161,7 +281,6 @@ def main(argv): entryArray.sort() - # Create the minidom document outDoc = xml.dom.minidom.Document() @@ -173,7 +292,7 @@ def main(argv): # start processing entries for entry in entryArray: - addEntryForId(outDoc, ljElement, username, entry) + addEntryForId(outDoc, ljElement, username, entry, includeSecure) currentFileEntry += 1