DEV.to integration (#11)

Co-authored-by: Joan Reyero <joan@crowd.dev>
Co-authored-by: anilb <epipav@gmail.com>

Author: 3 people

backend/package-lock.json

@@ -48,6 +48,7 @@
     "express-rate-limit": "6.5.1",
     "formidable-serverless": "1.1.1",
     "helmet": "4.1.1",
+    "html-to-text": "^8.2.1",
     "jsonwebtoken": "8.5.1",
     "lodash": "4.17.21",
     "meilisearch": "^0.26.0",
@@ -64,6 +65,7 @@
     "passport-slack": "0.0.7",
     "pg": "^8.7.3",
     "pm2": "^5.2.0",
+    "sanitize-html": "^2.7.1",
     "sequelize": "6.21.2",
     "sequelize-cli-typescript": "^3.2.0-c",
     "stripe": "10.0.0",
@@ -75,8 +77,10 @@
   "devDependencies": {
     "@babel/plugin-transform-runtime": "^7.18.10",
     "@babel/preset-env": "^7.16.10",
+    "@types/html-to-text": "^8.1.1",
     "@types/jest": "^27.4.0",
     "@types/node": "^17.0.21",
+    "@types/sanitize-html": "^2.6.2",
     "@typescript-eslint/eslint-plugin": "^5.17.0",
     "@typescript-eslint/parser": "^5.17.0",
     "copyfiles": "2.4.1",

backend/package.json

@@ -0,0 +1,14 @@
+import PermissionChecker from '../../../services/user/permissionChecker'
+import ApiResponseHandler from '../../apiResponseHandler'
+import Permissions from '../../../security/permissions'
+import IntegrationService from '../../../services/integrationService'
+
+export default async (req, res) => {
+  try {
+    new PermissionChecker(req).validateHas(Permissions.values.tenantEdit)
+    const payload = await new IntegrationService(req).devtoConnectOrUpdate(req.body)
+    await ApiResponseHandler.success(req, res, payload)
+  } catch (error) {
+    await ApiResponseHandler.error(req, res, error)
+  }
+}

backend/src/api/integration/helpers/devtoCreateOrUpdate.ts

@@ -0,0 +1,29 @@
+import PermissionChecker from '../../../services/user/permissionChecker'
+import Permissions from '../../../security/permissions'
+import ApiResponseHandler from '../../apiResponseHandler'
+import Error400 from '../../../errors/Error400'
+import { getUserByUsername } from '../../../serverless/integrations/usecases/devto/getUser'
+import { getOrganization } from '../../../serverless/integrations/usecases/devto/getOrganization'
+
+export default async (req, res) => {
+  try {
+    new PermissionChecker(req).validateHasAny([
+      Permissions.values.integrationCreate,
+      Permissions.values.integrationEdit,
+    ])
+
+    if (req.query.username) {
+      const result = await getUserByUsername(req.query.username)
+      await ApiResponseHandler.success(req, res, result)
+    } else if (req.query.organization) {
+      const result = await getOrganization(req.query.organization)
+      await ApiResponseHandler.success(req, res, result)
+    } else {
+      // throw bad request since we don't have either of the query params
+      await ApiResponseHandler.error(req, res, new Error400(req.language))
+    }
+  } catch (error) {
+    console.error(error)
+    await ApiResponseHandler.error(req, res, error)
+  }
+}

backend/src/api/integration/helpers/devtoValidators.ts

@@ -22,6 +22,8 @@ export default (app) => {
     `/discord-authenticate/:tenantId/:guild_id`,
     require('./helpers/discordAuthenticate').default,
   )
+  app.get('/tenant/:tenantId/devto-validate', require('./helpers/devtoValidators').default)
+  app.post('/tenant/:tenantId/devto-connect', require('./helpers/devtoCreateOrUpdate').default)
 
   if (getConfig().AUTH_SOCIAL_TWITTER_CLIENT_ID) {
     passport.use(getTwitterStrategy())

backend/src/api/integration/index.ts

@@ -477,6 +477,59 @@ describe('eagleEyeContentRepository tests', () => {
 
       await addAll(mockIRepositoryOptions)
 
+      const k1 = {
+        sourceId: 'sourceIdk1',
+        vectorId: 'sourceIdk1',
+        status: null,
+        platform: 'hacker_news',
+        title: 'title',
+        userAttributes: { github: 'hey', twitter: 'ho' },
+        text: 'text',
+        postAttributes: {
+          score: 10,
+        },
+        url: 'url',
+        timestamp: new Date(),
+        username: 'username',
+        keywords: ['keyword1'],
+        similarityScore: 0.9,
+      }
+
+      await new EagleEyeContentService(mockIRepositoryOptions).upsert(k1)
+
+      const k2 = {
+        sourceId: 'sourceIdk2',
+        vectorId: 'sourceIdk2',
+        status: null,
+        platform: 'hacker_news',
+        title: 'title',
+        userAttributes: { github: 'hey', twitter: 'ho' },
+        text: 'text',
+        postAttributes: {
+          score: 10,
+        },
+        url: 'url',
+        timestamp: new Date(),
+        username: 'username',
+        keywords: ['keyword2'],
+        similarityScore: 0.9,
+      }
+
+      try {
+        await EagleEyeContentRepository.findAndCountAll(
+          {
+            filter: {
+              keywords: 'keyword1,keyword2',
+            },
+          },
+          mockIRepositoryOptions,
+        )
+      } catch (e) {
+        console.log(e)
+      }
+
+      await new EagleEyeContentService(mockIRepositoryOptions).upsert(k2)
+
       expect(
         (
           await EagleEyeContentRepository.findAndCountAll(
@@ -488,7 +541,7 @@ describe('eagleEyeContentRepository tests', () => {
             mockIRepositoryOptions,
           )
         ).count,
-      ).toBe(2)
+      ).toBe(5)
     })
   })
 

backend/src/database/repositories/__tests__/eagleEyeContentRepository.test.ts

@@ -1,3 +1,4 @@
+import sanitizeHtml from 'sanitize-html'
 import lodash from 'lodash'
 import Sequelize from 'sequelize'
 import SequelizeRepository from './sequelizeRepository'
@@ -18,6 +19,14 @@ class ActivityRepository {
 
     const transaction = SequelizeRepository.getTransaction(options)
 
+    if (data.crowdInfo && data.crowdInfo.body) {
+      data.crowdInfo.body = sanitizeHtml(data.crowdInfo.body).trim()
+    }
+
+    if (data.crowdInfo && data.crowdInfo.title) {
+      data.crowdInfo.title = sanitizeHtml(data.crowdInfo.title).trim()
+    }
+
     const record = await options.database.activity.create(
       {
         ...lodash.pick(data, [
@@ -68,6 +77,14 @@ class ActivityRepository {
       throw new Error404()
     }
 
+    if (data.crowdInfo && data.crowdInfo.body) {
+      data.crowdInfo.body = sanitizeHtml(data.crowdInfo.body).trim()
+    }
+
+    if (data.crowdInfo && data.crowdInfo.title) {
+      data.crowdInfo.title = sanitizeHtml(data.crowdInfo.title).trim()
+    }
+
     record = await record.update(
       {
         ...lodash.pick(data, [

backend/src/database/repositories/activityRepository.ts

@@ -211,9 +211,10 @@ export default class EagleEyeContentRepository {
       }
 
       if (filter.keywords) {
+        // Overlap will take a post where any keyword matches any of the filter keywords
         whereAnd.push({
           keywords: {
-            [Op.contains]: filter.keywords.split(','),
+            [Op.overlap]: filter.keywords.split(','),
           },
         })
       }

backend/src/database/repositories/eagleEyeContentRepository.ts

@@ -145,14 +145,14 @@ describe('Dev.to iterator tests', () => {
       expect(activity.isKeyAction).toEqual(DevtoGrid.comment.isKeyAction)
 
       const activityCrowdInfo = activity.crowdInfo as any
-      expect(activityCrowdInfo.bodyHtml).toEqual('Hello world!')
+      expect(activityCrowdInfo.body).toEqual('Hello world!')
       expect(activityCrowdInfo.userUrl).toEqual('https://dev.to/johndoe')
-      expect(activityCrowdInfo.commentUrl).toEqual('https://dev.to/johndoe/comment/123')
+      expect(activityCrowdInfo.url).toEqual('https://dev.to/johndoe/comment/123')
       expect(activityCrowdInfo.articleUrl).toEqual(article.url)
+      expect(activityCrowdInfo.articleTitle).toEqual(article.title)
 
       const communityMember = activity.communityMember
       expect(communityMember.username[PlatformType.DEVTO]).toEqual('johndoe')
-      expect(communityMember.crowdInfo[PlatformType.DEVTO].id).toEqual(123)
       expect(communityMember.bio).toEqual('Nice profile you got there')
       expect(communityMember.location).toEqual('Venice, Italy')
       expect(communityMember.crowdInfo[PlatformType.TWITTER].url).toEqual(

backend/src/serverless/integrations/iterators/__tests__/devtoIterator.test.ts

@@ -1,7 +1,8 @@
 import lodash from 'lodash'
+import sanitizeHtml from 'sanitize-html'
 import { DevtoArticle, DevtoComment, DevtoUser } from '../usecases/devto/types'
 import { single } from '../../../utils/arrays'
-import { getUser } from '../usecases/devto/getUser'
+import { getUserById } from '../usecases/devto/getUser'
 import IntegrationRepository from '../../../database/repositories/integrationRepository'
 import { IRepositoryOptions } from '../../../database/repositories/IRepositoryOptions'
 import Operations from '../../dbOperations/operations'
@@ -96,8 +97,10 @@ export default class DevtoIterator extends BaseIterator {
     // does not contain all the user information we can get full profile and set it to comments
     const userIds: number[] = DevtoIterator.getUserIdsFromComments(comments)
     for (const userId of userIds) {
-      const fullUser = await getUser(userId)
-      DevtoIterator.setFullUser(comments, fullUser)
+      const fullUser = await getUserById(userId)
+      if (fullUser !== null) {
+        DevtoIterator.setFullUser(comments, fullUser)
+      }
     }
 
     return {
@@ -195,30 +198,33 @@ export default class DevtoIterator extends BaseIterator {
 
     const communityMember: CommunityMember = {
       username: {
-        [PlatformType.DEVTO]: comment.fullUser.username,
+        [PlatformType.DEVTO]: comment.user.username,
       },
       crowdInfo: {
         [PlatformType.DEVTO]: {
-          id: comment.fullUser.id,
+          url: `https://dev.to/${encodeURIComponent(comment.fullUser.username)}`,
         },
       },
-      bio: comment.fullUser?.summary || '',
-      location: comment.fullUser?.location || '',
     }
 
-    if (comment.fullUser.twitter_username) {
+    if (comment.user.twitter_username) {
       communityMember.crowdInfo.twitter = {
-        url: `https://twitter.com/${comment.fullUser.twitter_username}`,
+        url: `https://twitter.com/${comment.user.twitter_username}`,
       }
-      communityMember.username.twitter = comment.fullUser.twitter_username
+      communityMember.username.twitter = comment.user.twitter_username
     }
 
-    if (comment.fullUser.github_username) {
+    if (comment.user.github_username) {
       communityMember.crowdInfo.github = {
-        name: comment.fullUser.name,
-        url: `https://github.com/${comment.fullUser.github_username}`,
+        name: comment.user.name,
+        url: `https://github.com/${comment.user.github_username}`,
       }
-      communityMember.username.github = comment.fullUser.github_username
+      communityMember.username.github = comment.user.github_username
+    }
+
+    if (comment.fullUser) {
+      communityMember.bio = comment.fullUser?.summary || ''
+      communityMember.location = comment.fullUser?.location || ''
     }
 
     activities.push({
@@ -229,12 +235,15 @@ export default class DevtoIterator extends BaseIterator {
       sourceId: comment.id_code,
       sourceParentId: parentCommentId,
       crowdInfo: {
-        bodyHtml: comment.body_html,
-        userUrl: `https://dev.to/${encodeURIComponent(comment.fullUser.username)}`,
-        commentUrl: `https://dev.to/${encodeURIComponent(comment.fullUser.username)}/comment/${
+        body: sanitizeHtml(comment.body_html),
+        url: `https://dev.to/${encodeURIComponent(comment.fullUser.username)}/comment/${
           comment.id_code
         }`,
+        thread: parentCommentId !== undefined,
+
+        userUrl: `https://dev.to/${encodeURIComponent(comment.fullUser.username)}`,
         articleUrl: article.url,
+        articleTitle: article.title,
       },
       communityMember,