Add SAM assignment scraper (#51)

* Add node-html-parser

* Add SAM scraper

* Add documentation

* Integrate SAM scraper into bot

* Remove un-used variable professor

Author: Douile

.config.example.json

@@ -1,6 +1,7 @@
 {
   "api": "https://liverpool.instructure.com/api/v1",
   "course_filter": "^202021-(COMP[0-9]+)",
+  "sam_course_filter": "^COMP1",
   "prefix": "ca!",
   "question_directory": "questions/",
   "answer_directory": "answers/",

.env_SAMPLE

@@ -1,2 +1,3 @@
 BOT_TOKEN = YOUR_TOKEN_HERE
 CANVAS_TOKEN = YOUR_TOKEN_HERE
+SAM_TOKEN = YOUR_TOKEN_HERE

package-lock.json

@@ -34,6 +34,7 @@
     "discord.js-light": "^3.3.6",
     "dotenv": "^8.2.0",
     "node-fetch": "^2.6.1",
+    "node-html-parser": "^2.0.0",
     "simple-git": "^2.22.0",
     "sqlite3": "^5.0.0"
   },

package.json

@@ -2,6 +2,8 @@
 
 const fetch = require("node-fetch");
 
+const { USER_AGENT } = require('./constants.js');
+
 const PAGE_LENGTH = 50;
 
 const RE_LINKURL = /<([^>]+)>/g;
@@ -13,6 +15,7 @@ const req = Object.freeze({
   'get': async function(url, auth, headers) {
     if (!headers) headers = {};
     headers['Authorization'] = `Bearer ${auth}`;
+    if (!('User-Agent' in headers)) headers['User-Agent'] = USER_AGENT;
     console.log('GET', String(url));
     const res = await fetch(url, {headers});
     bucket.remaining = parseFloat(res.headers.get('X-Rate-Limit-Remaining'));

src/canvas.js

@@ -20,8 +20,9 @@ function parseAssignments(week, assignments, courses) {
 }
 
 class CanvasUtils {
-  constructor(canvas, overrides) {
+  constructor(canvas, sam, overrides) {
     this.canvas = canvas;
+    this.sam = sam;
     this.overrides = overrides;
     this.lastUpdate = undefined;
     this.cache = undefined;
@@ -34,7 +35,14 @@ class CanvasUtils {
       promises.push(this.canvas.getCourseAssignments(courseID));
       promises.push(this.canvas.getCourseDiscussions(courseID));
     }
-    const assignments = (await Promise.all(promises)).flat();
+    let assignments = (await Promise.all(promises)).flat();
+    const samCourses = await this.sam.getFilteredCourses();
+    promises = [];
+    for (let courseID of samCourses) {
+      promises.push(this.sam.getCourseAssignments(courseID));
+      courses[courseID] = courseID;
+    }
+    assignments = assignments.concat((await Promise.all(promises)).flat());
     return { courses, assignments };
   }
 

src/canvasutils.js

@@ -12,3 +12,16 @@ exports.BOT_PRESENCE = Object.freeze({ status: 'online', activity: { type: 'WATC
 exports.DB_NAME = '.store.db';
 
 exports.EMBED_COLOR = 0xff0000;
+
+exports.USER_AGENT = 'DiscordCanvasBot/1.0';
+
+exports.OPTS_HTML_PARSE = Object.freeze({
+  lowerCaseTagNames: false,
+  comment: false,
+  blockTextElements: {
+    script: false,
+    noscript: false,
+    style: false,
+    pre: true
+  }
+});

src/constants.js

@@ -5,6 +5,7 @@ const { Client } = require('discord.js-light');
 const sqlite3 = require('sqlite3');
 
 const Canvas = require('./canvas.js');
+const Sam = require('./sam.js');
 const CanvasUtils = require('./canvasutils.js');
 const { assignmentAutoUpdate } = require('./autoupdate.js');
 const { asyncWrap, unwrapSync, ipcSend } = require('./utils.js');
@@ -98,13 +99,14 @@ function awaitOpen(database) {
   });
 }
 
-const startBot = module.exports = async function(botToken, canvasToken, config) {
+const startBot = module.exports = async function(botToken, canvasToken, samToken, config) {
   const db = new sqlite3.Database(DB_NAME, sqlite3.OPEN_READWRITE);
   await awaitOpen(db);
   db.on('error', console.error);
   config = enforceType(Config, config);
   const canvas = new Canvas(canvasToken, config);
-  const canvasUtils = new CanvasUtils(canvas, config.overrides);
+  const sam = new Sam(samToken, config);
+  const canvasUtils = new CanvasUtils(canvas, sam, config.overrides);
   Object.defineProperties(client, {
     config: {
       configurable: false,
@@ -194,10 +196,11 @@ function shutdown() {
 if (require.main === module) {
   const DISCORD_TOKEN = process.env.DISCORD_TOKEN || process.env.BOT_TOKEN || '';
   const CANVAS_TOKEN = process.env.CANVAS_TOKEN || '';
+  const SAM_TOKEN = process.env.SAM_TOKEN || '';
   const CONFIG = require('../.config.json');
 
   hasShutdown = false;
-  startBot(DISCORD_TOKEN, CANVAS_TOKEN, CONFIG).then(null, function() {
+  startBot(DISCORD_TOKEN, CANVAS_TOKEN, SAM_TOKEN, CONFIG).then(null, function() {
     console.error.apply(this, arguments);
     process.exit(1);
   });

src/index.js

@@ -0,0 +1,104 @@
+'use strict';
+
+const fetch = require("node-fetch");
+const HTMLParser = require('node-html-parser');
+
+const { USER_AGENT, OPTS_HTML_PARSE } = require('./constants.js');
+
+const URL = 'https://sam.csc.liv.ac.uk/COMP/CW_List.pl';
+const AUTH_COOKIE = '/var/www/SAM/CJar/InterViewMgr_COMP';
+
+const RE_NBSP = /&nbsp/g;
+
+const req = async function(url, {sessionCookie, method, body, mimeType }) {
+  return await fetch(url, {
+    method, body, headers: {
+      'User-Agent': USER_AGENT,
+      'Cookie': `${encodeURIComponent(AUTH_COOKIE)}=${encodeURIComponent(sessionCookie)}`,
+      'Content-Type': mimeType
+    }
+  });
+}
+
+const get = function(url, sessionCookie) {
+  return req(url, {sessionCookie, method: 'GET'});
+}
+
+const post = function(url, sessionCookie, body) {
+  return req(url, {sessionCookie, method: 'POST', body, mimeType: 'application/x-www-form-urlencoded' });
+}
+
+
+class Sam {
+  constructor(token, options) {
+    if (typeof token !== 'string' || token.length === 0) throw new Error('Invalid API token');
+    this.token = token;
+    this.options = options || {};
+  }
+}
+
+/**
+* getCourses
+* fetch and parse a list available courses
+* @param {String} sessionCookie - Your SAM login session cookie
+* @returns {Array.<String>} list of module names
+*/
+const getCourses = async function() {
+  const res = await get(URL, this.token);
+  const text = await res.text();
+
+  const html = HTMLParser.parse(text, OPTS_HTML_PARSE);
+
+  const moduleSelect = html.querySelector('select[NAME="qryModule"]');
+  if (moduleSelect === null) throw new Error('Could not find modules');
+
+  const moduleElements = moduleSelect.querySelectorAll('OPTION');
+
+  const modules = moduleElements.map(el => el.getAttribute('VALUE'));
+  return modules;
+}
+Sam.prototype.getCourses = getCourses;
+
+
+/**
+* getFilteredCourses
+* fetch a list of available courses and filter them
+*/
+const getFilteredCourses = async function() {
+  const courses = await this.getCourses();
+  const regex = new RegExp(this.options.sam_course_filter, 'g');
+  return courses.filter(course => course.match(regex) !== null);
+}
+Sam.prototype.getFilteredCourses = getFilteredCourses;
+
+
+/**
+* getCourseAssignments
+* fetch and parse the list of assignmens for given module
+* @param {String} sessionCookie - Your SAM login session cookie
+* @param {String} course - Name of module
+* @returns {Array.<Assignment>} assignments for given module
+*/
+const getCourseAssignments = async function(course) {
+  const res = await post(URL, this.token, `qryModule=${encodeURIComponent(course)}`);
+  const text = await res.text();
+
+  const html = HTMLParser.parse(text, OPTS_HTML_PARSE);
+
+  const assignmentRows = html.querySelectorAll('TABLE.general TR');
+
+  const assignmentData = assignmentRows.map(el => el.querySelectorAll('TD')).filter(d => d.length === 4);
+
+  return assignmentData.map(td => {
+    const [course] = td[0].structuredText.split('\n');
+    const [id, name] = td[1].structuredText.split('\n');
+    const due = td[2].structuredText;
+    const a = td[3].querySelector('A');
+    const url = a === null ? undefined : a.getAttribute('href');
+    const dueDate = new Date(due);
+    return { id: `SAM-${course}-${id}`, name: name.replace(RE_NBSP,'').trim(), course, due: dueDate.getTime(), dueDate, points: 1, url };
+  })
+}
+Sam.prototype.getCourseAssignments = getCourseAssignments;
+
+module.exports = Sam;

src/sam.js

@@ -139,6 +139,7 @@ const ConfigAutomatedAssignment = Object.freeze({
 const Config = Object.freeze({
   api: TypeOrDefault(String, 'https://instructure.com/api/v1'),
   course_filter: TypeOrDefault(String, '(.*)'),
+  sam_course_filter: TypeOrDefault(String, '.'),
   question_directory: TypeOrDefault(String, "questions/"),
   answer_directory: TypeOrDefault(String, "answers/"),
   prefix: TypeOrDefault(String, 'ca!'),