For Dropbox handling I chose a pretty small library, node-dropbox. To use it, I went in Dropbox's developer dashboard and created an access token (instead of using secrets and keys) and saved that in my .env. Then onto the helper:
const dropbox = require('node-dropbox').api(process.env.ACCESS_TOKEN)
function Dropbox() {}
Dropbox.getFolder = function(path) {
return new Promise(function(resolve) {
// This will get all the metadata for each file from the folder.
dropbox.getMetadata(path, function(error, result, folder) {
if (error) {
console.log('Getting ' + path + ' failed.')
}
else {
resolve(folder)
}
})
})
}
Dropbox.getFile = function(path) {
return new Promise(function(resolve) {
// This will return the contents of a file.
dropbox.getFile(path, function(error, result, file) {
if (error) {
console.log('Getting ' + path + ' failed.')
}
else {
resolve(file)
}
})
})
}Lastly, we will need three new db methods that we'll use when syncing:
Db.updatePost = function(post) {
const query = 'UPDATE posts SET' +
' ' + fields() +
' = ' + values(post) + ' WHERE ' +
' link = \'' + post.link + '\' AND ' +
' datetime = \'' + post.datetime + '\''
// Connect and run the query
})
Db.insertPost = function(post) {
const query = 'INSERT INTO posts ' +
fields() +
' VALUES ' + values(post)
// Connect and run the query
})
Db.delete = function(post) {
const query = 'DELETE FROM posts WHERE' +
' link = \'' + post.link + '\' AND' +
' datetime = \'' + post.datetime + '\''
// Connect and run the query
})The fields and values methods are just helpers for building the query, one for returning the fields, one for creating the values:
function fields() {
return '(title, body, datetime, modified, link, readingtime)'
}
function values(post) {
// Escape all ' in the title and body by adding another one
const title = post.title.replace(new RegExp('\'', 'g'), '\'\'')
const body = post.body.replace(new RegExp('\'', 'g'), '\'\'')
return '(\'' + title + '\', ' +
'\'' + body + '\', ' +
'\'' + post.datetime + '\', ' +
'\'' + post.modified + '\', ' +
'\'' + post.link + '\', ' +
'\'' + post.readingTime + '\')'
// This will return a string similar to fields:
('A title', 'A body', '2016-07-22-2148', 'a-title', '2 min')
}And these two come together in the sync route:
// Force means update even if the date of the file is the same as the database post's
// shouldDelete means files have been deleted and the respective posts need to be deleted
// This is because for deleting there's an extra couple of iterations over all posts, and takes a bit longer
router.get('/' + process.env.MY_SYNC_KEY + '/:key1?/:key2?', function(req, res) {
const shouldDelete = req.params.key1 == 'delete' || req.params.key2 == 'delete'
const forced = req.params.key1 == 'force' || req.params.key2 == 'force'This will wait for both promises to finish, then return them in an array:
Promise.all([Db.fetchPostsForUpdating(), Dropbox.getFolder('/posts')]).then(function(data) {
let posts = data[0].posts
const folder = data[1]Since Dropbox.getFile is async, and wrapped in a promise, we need to wait for all of them to finish. Luckily, bluebird (Promise) has a nice little helper, map, which will do just that; we just have to return a Promise:
Promise.map(folder.contents, function(item) {
return Promise.join(item, Dropbox.getFile(item.path), function(item, file) {
return { item: item, file: file }
})This will return an array of promises containing the item and the file contents, so we map once more, to eventually create an array of Posts:
}).map(function(dropboxData) {
const item = dropboxData.item
const file = dropboxData.fileFirst, we create a safe-ish link out of the file name:
const matches = item.path.match(/\/(posts)\/(\d{4})-(\d{2})-(\d{2})-(\d{4})-([\w\s\.\/\}\{\[\]_#&@$:"';,!=\?\+\*\-\)\(]+)\.md$/)
const datetime = matches[2] + '-' + matches[3] + '-' + matches[4] + '-' + matches[5]
let link = matches[6].replace(/([#,;!:"\'\?\[\]\{\}\(\$\/)]+)/g, '')
.replace(/&/g, 'and')
.replace(/\s|\./g, '-')
.toLowerCase()Then we create the title from the first line, delete the second line and create the body out of the rest by transforming the markdown syntax in html with marked, save the last modified time in modified and return the Post object:
let lines = file.toString().split('\n')
const title = lines[0]
lines.splice(0, 2)
const body = lines.join('\n')
const modified = item.client_mtime
return new Post(title, marked(body), Post.readingTime(body),
datetime, modified, link)We now have an array of Posts that represent the data in Dropbox (newPosts), and an array of Posts that represent the data in the database (posts):
}).then(function(newPosts) {
if (shouldDelete) {
// Iterate through existing posts, and if no corresponding
// file is found, delete the post, and remove it from the data.
data[0].posts.forEach(function(post, index) {
let matchingNewPosts = newPosts.filter(function(newPost) {
return Post.linksMatch(newPost, post) &&
newPost.datetime == post.datetime
})
if (matchingNewPosts.length) { return }
// We don't need to wait for this to finish
Db.deletePost(post)
// After the iteration is finished, posts will represent the data in the database
// with the extra posts removed.
posts.splice(index, 1)
})
}Quick explanation: if a Dropbox post has the same link and the same datetimeas a database post , it means it's the same entity, since these fields are created out of filenames, which are unique; if they have the same link, but different datetimes, then they are different entities.
We will now iterate through the Dropbox posts, find the ones with matching links, and if none is found, we create it, but since we want to wait for all posts to be created, we will use bluebird's map again, and return an object that contains the matching posts (or empty array), and the newPost:
Promise.map(newPosts, function(newPost) {
// Just the one(s) with the same link
let matchingPosts = posts.filter(function(p) {
return Post.linksMatch(newPost, p)
})
let returnObject = {
newPost: newPost,
matchingPosts: matchingPosts
}
// Create
if (matchingPosts.length == 0) {
Db.createPost(newPost)
}
return returnObjectQuick reminder that when map finishes, we will have an array of objects:
}).each(function(data) {
const newPost = data.newPost
const matchingPosts = data.matchingPostsInside map we created new Posts if no matches were found. Here, if we have one or more matches, we will update the ones with matching datetime:
matchingPosts.forEach(function(matchingPost) {
// Update
if (newPost.datetime == matchingPost.datetime) {
// Only if these differ, no reason to query the db for nothing
if (newPost.modified != matchingPost.modified || forced) {
Db.updatePost(newPost)
}
return
}If any of the matches have non-matching datetimes, that means it's just another post, created by mistake with the same name, so we will append --1 to it, or if that exists --2, and so on:
let variant
// Create a new one, with same link, but duplicated.
// If it has --1 already, make it --2, and so on.
if (matchingPost.link.slice(-3, -1) == '--') {
variant = parseInt(matchingPost.link.slice(-1)[0])
}
else if (matchingPost.link.slice(-4, -2) == '--') {
variant = parseInt(matchingPost.link.slice(-2))
}
else {
variant = 0
}
variant += 1
newPost.link += '--' + variant
Db.createPost(newPost)
})When everything is finished, we just redirect back to the homepage:
}).then(function() {
res.redirect('/')
})
})
}).catch(function(error) {
console.log(error)
})
})Hopefully I managed to split and explain the code properly. If I didn't, don't hesitate to drop by and ask questions, or give any kind of feedback @rolandleth.