Skip to content

Commit

Permalink
🚀 Add url to json
Browse files Browse the repository at this point in the history
  • Loading branch information
Drew McMillan committed Apr 13, 2018
1 parent bef729f commit d28568e
Show file tree
Hide file tree
Showing 4 changed files with 5 additions and 6 deletions.
2 changes: 1 addition & 1 deletion cli.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
const yargs = require('yargs')
const lightcrawler = require('.')

const options = yargs.demandOption(['c', 'u'])
const options = yargs.demandOption(['c'])
.alias('u', 'url').describe('url', 'URL to crawl')
.alias('h', 'help').help('h')
.alias('c', 'config').describe('config', 'Options for lighthouse')
Expand Down
1 change: 1 addition & 0 deletions example.json
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
{
"url": "https://www.example.com",
"maxDepth": 1,
"maxChromeInstances": 5,
"limit": "/music/",
Expand Down
4 changes: 1 addition & 3 deletions index.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,12 @@ const stats = {
}

module.exports = (options) => {
console.log("ô¿ô light-mc-crawler has started crawling. If it looks like nothing is happening, wait, it is :)");

stats.startTime = new Date()

const configPath = path.resolve(options.config)
const config = JSON.parse(fs.readFileSync(configPath))

const crawler = new Crawler(options.url)
const crawler = new Crawler(options.url || config.url)
crawler.respectRobotsTxt = false
crawler.parseHTMLComments = false
crawler.parseScriptTags = false
Expand Down
4 changes: 2 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"name": "light-mc-crawler",
"version": "1.3.0",
"version": "2.0.0",
"description": "",
"main": "index.js",
"scripts": {
"example": "node ./cli.js --url https://www.example.com --config example.json"
"example": "node ./cli.js --config example.json"
},
"bin": {
"light-mc-crawler": "./cli.js"
Expand Down

0 comments on commit d28568e

Please sign in to comment.