isso: migrate: Handle deleted comments in Disqus migration (#994)

* isso: migrate: Handle deleted comments in Disqus migration

Fixes #979

Author: pkvach

CHANGES.rst

@@ -26,12 +26,14 @@ Bugfixes & Improvements
 - Make language code handling more robust (`#983`_, ix5)
 - Prevent auto creation of invalid links in comments (`#995`_, pkvach)
 - Fix W3C Validation issues (`#999`_, pkvach)
+- Handle deleted comments in Disqus migration (`#994`_, pkvach)
 
 .. _#951: https://github.com/posativ/isso/pull/951
 .. _#967: https://github.com/posativ/isso/pull/967
 .. _#983: https://github.com/posativ/isso/pull/983
 .. _#995: https://github.com/isso-comments/isso/pull/995
 .. _#999: https://github.com/isso-comments/isso/pull/999
+.. _#994: https://github.com/isso-comments/isso/pull/994
 
 0.13.1.dev0 (2023-02-05)
 ------------------------

isso/db/comments.py

@@ -38,7 +38,7 @@ def __init__(self, db):
             'CREATE TABLE IF NOT EXISTS comments (',
             '    tid REFERENCES threads(id), id INTEGER PRIMARY KEY, parent INTEGER,',
             '    created FLOAT NOT NULL, modified FLOAT, mode INTEGER, remote_addr VARCHAR,',
-            '    text VARCHAR, author VARCHAR, email VARCHAR, website VARCHAR,',
+            '    text VARCHAR NOT NULL, author VARCHAR, email VARCHAR, website VARCHAR,',
             '    likes INTEGER DEFAULT 0, dislikes INTEGER DEFAULT 0, voters BLOB NOT NULL,',
             '    notification INTEGER DEFAULT 0);'])
         try:

isso/migrate.py

@@ -92,10 +92,11 @@ def migrate(self):
         for post in tree.findall(Disqus.ns + 'post'):
             email = post.find('{0}author/{0}email'.format(Disqus.ns))
             ip = post.find(Disqus.ns + 'ipAddress')
+            comment_text = post.find(Disqus.ns + 'message').text or ''
 
             item = {
                 'dsq:id': post.attrib.get(Disqus.internals + 'id'),
-                'text': post.find(Disqus.ns + 'message').text,
+                'text': comment_text,
                 'author': post.find('{0}author/{0}name'.format(Disqus.ns)).text,
                 'email': email.text if email is not None else '',
                 'created': mktime(strptime(
@@ -146,11 +147,13 @@ def migrate(self):
                     continue
 
                 email = post.find("{0}author/{0}email".format(Disqus.ns))
+                comment_text = post.find(Disqus.ns + 'message').text or ''
+
                 print(" * {0} by {1} <{2}>".format(
                     post.attrib.get(Disqus.internals + "id"),
                     post.find("{0}author/{0}name".format(Disqus.ns)).text,
                     email.text if email is not None else ""))
-                print(textwrap.fill(post.find(Disqus.ns + "message").text,
+                print(textwrap.fill(comment_text,
                                     initial_indent="  ", subsequent_indent="  "))
                 print("")
 

isso/tests/disqus.xml

@@ -113,4 +113,37 @@
         <thread dsq:id="7" />
     </post>
 
+    <!-- deleted post -->
+    <post dsq:id="9">
+        <id/>
+        <message/>
+        <createdAt>2013-10-10T19:20:29Z</createdAt>
+        <isDeleted>true</isDeleted>
+        <isSpam>false</isSpam>
+        <author>
+            <email>foo@bar.com</email>
+            <name>peter</name>
+            <isAnonymous>true</isAnonymous>
+        </author>
+        <ipAddress>127.0.0.1</ipAddress>
+        <thread dsq:id="2" />
+    </post>
+
+    <!-- reference to previous deleted post -->
+    <post dsq:id="11">
+        <message><![CDATA[<p>Hello, World.</p>]]></message>
+        <createdAt>2013-10-11T06:52:33Z</createdAt>
+        <isDeleted>false</isDeleted>
+        <isSpam>false</isSpam>
+        <author>
+            <email>foo@bar.com</email>
+            <name>user</name>
+            <isAnonymous>false</isAnonymous>
+            <username>user</username>
+        </author>
+        <ipAddress>127.0.0.1</ipAddress>
+        <thread dsq:id="2" />
+        <parent dsq:id="9" />
+    </post>
+
   </disqus>

isso/tests/test_migration.py

@@ -56,7 +56,7 @@ def test_disqus_empty_id_workaround(self):
         Disqus(db, xml, empty_id=True).migrate()
 
         self.assertEqual(
-            len(db.execute("SELECT id FROM comments").fetchall()), 3)
+            len(db.execute("SELECT id FROM comments").fetchall()), 5)
 
         self.assertEqual(db.threads["/"]["title"], "Hello, World!")
         self.assertEqual(db.threads["/"]["id"], 1)
@@ -67,9 +67,12 @@ def test_disqus_empty_id_workaround(self):
         self.assertEqual(a["email"], "foo@bar.com")
         self.assertEqual(a["remote_addr"], "127.0.0.0")
 
-        b = db.comments.get(2)
+        b = db.comments.get(3)
         self.assertEqual(b["parent"], a["id"])
 
+        deleted_comment = db.comments.get(2)
+        self.assertEqual(deleted_comment["text"], "")
+
     def test_wordpress(self):
 
         xml = join(dirname(__file__), "wordpress.xml")