migrate airtable attachments

Author: wslyvh

.github/workflows/orgs-import.yml

@@ -0,0 +1,32 @@
+name: Import Org logos
+
+on:
+  schedule:
+    - cron: "0 0 * * *"
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@master
+
+      - name: Use Node.js
+        uses: actions/setup-node@v1
+        with:
+          node-version: '16.x'
+
+      - name: Install dependencies
+        run: yarn install
+
+      - name: Import Pretalx schedule
+        run: yarn org:import
+        env:
+          PRETALX_API_KEY: ${{ secrets.PRETALX_API_KEY }}
+
+      - name: Commit updates
+        uses: EndBug/add-and-commit@v9
+        with:
+          default_author: github_actions
+          message: 'CI Logo import'

package.json

@@ -11,6 +11,7 @@
     "latest": "ts-node src/tools/latest",
     "latest:resources": "ts-node src/tools/latest-resources",
     "latest:issues": "ts-node src/tools/latest-issues",
+    "org:import": "ts-node src/tools/org-import",
     "twitter:gm": "ts-node src/tools/twitter-gm",
     "twitter:last-week": "ts-node src/tools/twitter-last-week",
     "twitter:popular": "ts-node src/tools/twitter-popular",

src/components/form/company.tsx

@@ -61,6 +61,8 @@ export function CompanyForm(props: Props) {
           readOnly={!!props.org.id}
         />
 
+        {/* 
+        // Remove Logo from Airtable/attachments
         <FormInput
           id="logo"
           name="Logo Url"
@@ -69,7 +71,7 @@ export function CompanyForm(props: Props) {
           value={props.org.logo}
           onChange={(value) => props.onChange({ ...props.org, logo: value })}
           readOnly={!!props.org.id}
-        />
+        /> */}
 
         <FormInput
           id="website"

src/services/airtable.ts

@@ -27,11 +27,12 @@ export class AirtableService {
       twitter: org.twitter,
       github: org.github,
       externalBoardUrl: org.externalBoardUrl,
-      logo: [
-        {
-          url: org.logo,
-        } as any,
-      ],
+      // Remove Logo from Airtable/attachments
+      // logo: [
+      //   {
+      //     url: org.logo,
+      //   } as any,
+      // ],
     })
 
     if (!response.id) {

src/services/jobs.ts

@@ -1,3 +1,4 @@
+import * as dotenv from 'dotenv'
 import Airtable, { Record, FieldSet } from 'airtable'
 import { Organization } from 'types/org'
 import { Job } from 'types/job'
@@ -13,6 +14,9 @@ import {
 } from './jobs/index'
 import { defaultSlugify, isEmail } from 'utils/helpers'
 import { getJobTags } from 'utils/jobs'
+import { SITE_URL } from 'utils/constants'
+
+dotenv.config()
 
 if (!process.env.AIRTABLE_API_KEY || !process.env.AIRTABLE_API_KEY) {
   throw new Error('Airtable API Base or Key not set.')
@@ -168,14 +172,17 @@ export function toOrganization(source: Record<FieldSet>): Organization {
     recordId: source.fields['recordId'],
   } as Organization
 
-  if (source.fields['logo'] && Array.isArray(source.fields['logo']) && (source.fields['logo'] as any[]).length > 0) {
-    org.logo = (source.fields['logo'] as any[])[0].url
-  }
+  // Remove Logo from Airtable/attachments
+  // if (source.fields['logo'] && Array.isArray(source.fields['logo']) && (source.fields['logo'] as any[]).length > 0) {
+  //   org.logo = (source.fields['logo'] as any[])[0].url
+  // }
   if (source.fields['website']) {
     org.website = (source.fields['website'] as string) ?? ''
   }
   if (source.fields['twitter']) {
-    org.twitter = (source.fields['twitter'] as string) ?? ''
+    const handle = (source.fields['twitter'] as string) ?? ''
+    org.twitter = handle
+    org.logo = `${SITE_URL}assets/orgs/${handle.replace('@', '')}.png`
   }
   if (source.fields['github']) {
     org.github = (source.fields['github'] as string) ?? ''

src/tools/org-import.ts

@@ -0,0 +1,45 @@
+import * as dotenv from 'dotenv'
+import fs from 'fs'
+import fetch from 'cross-fetch'
+import path from 'path'
+import { GetOrganizations } from '../services/jobs'
+
+dotenv.config()
+
+console.log('Import org logos')
+run()
+
+const dir = path.join(__dirname, `../../public/assets/orgs`)
+fs.mkdir(dir, { recursive: true }, () => '')
+
+async function run() {
+  const orgs = await GetOrganizations()
+  const handles = orgs.map((i) => i.twitter?.replaceAll('@', '')).filter((i) => !!i) as string[]
+
+  console.log('Orgs', orgs.length)
+  console.log('Handles', handles.length)
+
+  Promise.all(handles.map((i) => fetchAndSave(i))).then((results) => {
+    const total = results.length
+
+    console.log(`Total: ${total}`)
+    console.log(`Failed: ${results.filter((i) => i === false).length}`)
+  })
+}
+
+async function fetchAndSave(handle: string) {
+  // console.log('Fetching', handle)
+
+  try {
+    const response = await fetch(`https://unavatar.io/twitter/${handle}`)
+    const blob = await response.blob()
+    const arrayBuffer = await blob.arrayBuffer()
+    const buffer = Buffer.from(arrayBuffer)
+    fs.writeFileSync(`${dir}/${handle}.png`, buffer)
+    return true
+  } catch (e) {
+    console.log('Unable to fetch', handle)
+    console.error(e)
+    return false
+  }
+}

tsconfig.json

@@ -16,6 +16,9 @@
     "incremental": true,
     "baseUrl": "./src",
   },
+  "ts-node": {
+    "require": ["tsconfig-paths/register"]
+  },
   "include": ["next-env.d.ts", "src"],
   "exclude": ["node_modules"]
 }