From 6ff2ccf932d6253db3d15a12bfd705b70d9e27e4 Mon Sep 17 00:00:00 2001 From: Anton Date: Sat, 6 Apr 2019 04:29:55 +0200 Subject: [PATCH] Move archive functionality to mbjs-archive --- CHANGELOG.md | 1 + package-lock.json | 43 ++++++++++++++ package.json | 1 + src/index.js | 2 +- src/lib/{archives/index.js => archives.js} | 29 +++++++--- src/lib/archives/create.js | 67 ---------------------- src/lib/archives/read.js | 41 ------------- 7 files changed, 66 insertions(+), 118 deletions(-) rename src/lib/{archives/index.js => archives.js} (79%) delete mode 100644 src/lib/archives/create.js delete mode 100644 src/lib/archives/read.js diff --git a/CHANGELOG.md b/CHANGELOG.md index a3382c7..629e050 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Moved archive functionality to [mbjs-archive](https://gitlab.rlp.net/motionbank/mbjs/archive) module - Updated [mbjs-data-models](https://gitlab.rlp.net/motionbank/mbjs/data-models) to 0.1.2 ([release_0_1](https://gitlab.rlp.net/motionbank/mbjs/data-models/commits/release_0_1) branch) diff --git a/package-lock.json b/package-lock.json index fa245b6..71702b2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2322,6 +2322,49 @@ "@arr/every": "^1.0.0" } }, + "mbjs-archive": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/mbjs-archive/-/mbjs-archive-1.0.0.tgz", + "integrity": "sha512-h/IEh5oAxZvJNLv9TIc/sg8nr2Od5NvnLHWW/9+PqFn768aXc7ERAB5hIozqmdSPKThsy7v1P47ZexSMaRw0EA==", + "requires": { + "mbjs-utils": "0.0.6", + "mz": "^2.7.0", + "rimraf": "^2.6.3", + "yauzl": "^2.10.0", + "yazl": "^2.5.1" + }, + "dependencies": { + "glob": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.3.tgz", + "integrity": "sha512-vcfuiIxogLV4DlGBHIUOwI0IbrJ8HWPc4MU7HzviGeNho/UJDfi6B5p3sHeWIQ0KGIU0Jpxi5ZHxemQfLkkAwQ==", + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "rimraf": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz", + "integrity": "sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==", + "requires": { + "glob": "^7.1.3" + } + }, + "yazl": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/yazl/-/yazl-2.5.1.tgz", + "integrity": "sha512-phENi2PLiHnHb6QBVot+dJnaAZ0xosj7p3fWl+znIjBDlnMI2PsZCJZ306BPTFOaHf5qdDEI8x5qFrSOBN5vrw==", + "requires": { + "buffer-crc32": "~0.2.3" + } + } + } + }, "mbjs-data-models": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/mbjs-data-models/-/mbjs-data-models-0.1.2.tgz", diff --git a/package.json b/package.json index 894e19a..92cccbd 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "axios": "^0.18.0", "config": "^2.0.1", "luxon": "^1.3.3", + "mbjs-archive": "^1.0.0", "mbjs-data-models": "0.1.2", "mbjs-generic-api": "0.8.4", "mbjs-persistence": "1.0.0", diff --git a/src/index.js b/src/index.js index 5d39195..e6919f2 100644 --- a/src/index.js +++ b/src/index.js @@ -46,7 +46,7 @@ const setup = async function () { * Configure archives */ - const { setupArchives } = require('./lib/archives') + const setupArchives = require('./lib/archives') setupArchives(api, maps, annotations) await api.start() diff --git a/src/lib/archives/index.js b/src/lib/archives.js similarity index 79% rename from src/lib/archives/index.js rename to src/lib/archives.js index 5c086a1..79da94e 100644 --- a/src/lib/archives/index.js +++ b/src/lib/archives.js @@ -1,9 +1,13 @@ const os = require('os'), + path = require('path'), + fs = require('mz/fs'), multer = require('multer'), send = require('@polka/send-type'), - createArchive = require('./create'), - readArchive = require('./read') + config = require('config'), + Minio = require('minio'), + { ObjectUtil } = require('mbjs-utils'), + { read, write } = require('mbjs-archive') const setupArchives = function (api, mapService, annotationService) { const upload = multer({ dest: os.tmpdir() }) @@ -42,12 +46,23 @@ const setupArchives = function (api, mapService, annotationService) { } } - const url = await createArchive(api, data) + const + dir = path.join(os.tmpdir(), `archive_${ObjectUtil.slug(data.map.title)}_${data.map.uuid}`), + archivePath = await write(dir, data) + + const opts = Object.assign({}, config.assets.client) + opts.useSSL = config.assets.client.useSSL && (config.assets.client.useSSL === true || config.assets.client.useSSL === 'true') + opts.port = config.assets.client.port ? parseInt(config.assets.client.port) : undefined + const minioClient = new Minio.Client(opts) + await minioClient.fPutObject(config.assets.archivesBucket, path.basename(archivePath), archivePath, { 'Content-Type': 'application/zip' }) + await fs.unlink(archivePath) + const url = await minioClient.presignedGetObject(config.assets.archivesBucket, path.basename(archivePath)) + send(res, 200, url) }) api.app.post('/archives/maps/upload', async function (req, res) { upload.single('file')(req, res, async () => { - const results = await readArchive(req.file.path) + const results = await read(req.file.path) const copy = typeof req.body.title === 'string', overrideAuthor = req.body.overrideAuthor === 'true' @@ -131,8 +146,4 @@ const setupArchives = function (api, mapService, annotationService) { }) } -module.exports = { - setupArchives, - createArchive, - readArchive -} +module.exports = setupArchives diff --git a/src/lib/archives/create.js b/src/lib/archives/create.js deleted file mode 100644 index 942c86c..0000000 --- a/src/lib/archives/create.js +++ /dev/null @@ -1,67 +0,0 @@ -const - yazl = require('yazl'), - path = require('path'), - fs = require('mz/fs'), - rimraf = require('rimraf'), - os = require('os'), - config = require('config'), - Minio = require('minio'), - { Assert, ObjectUtil } = require('mbjs-utils') - -const createArchive = async function (api, data) { - Assert.isType(data.map, 'object', 'data.map must be object') - Assert.ok(Array.isArray(data.annotations), 'data.annotations must be array') - - const dir = path.join(os.tmpdir(), `archive_${ObjectUtil.slug(data.map.title)}_${data.map.uuid}`) - - await new Promise((resolve, reject) => { - rimraf(dir, err => { - if (err) { - api.captureException(err) - return reject(err) - } - resolve() - }) - }) - - const archive = new yazl.ZipFile() - - await fs.mkdir(dir) - await fs.mkdir(path.join(dir, 'maps')) - await fs.mkdir(path.join(dir, 'annotations')) - - const mapfile = path.join('maps', `${data.map.uuid}.json`) - await fs.writeFile(path.join(dir, mapfile), JSON.stringify(data.map)) - archive.addFile(path.join(dir, mapfile), mapfile) - - for (let a of data.annotations) { - const annofile = path.join('annotations', `${a.uuid}.json`) - await fs.writeFile(path.join(dir, annofile), JSON.stringify(a)) - archive.addFile(path.join(dir, annofile), annofile) - } - - archive.end() - - const archivePath = `${dir}.zip` - await new Promise((resolve, reject) => { - archive.outputStream.pipe(fs.createWriteStream(archivePath)) - .on('error', err => { - reject(err) - }) - .on('close', () => { - resolve() - }) - }) - - const opts = Object.assign({}, config.assets.client) - opts.useSSL = config.assets.client.useSSL && (config.assets.client.useSSL === true || config.assets.client.useSSL === 'true') - opts.port = config.assets.client.port ? parseInt(config.assets.client.port) : undefined - const minioClient = new Minio.Client(opts) - await minioClient.fPutObject(config.assets.archivesBucket, path.basename(archivePath), archivePath, { 'Content-Type': 'application/zip' }) - await fs.unlink(archivePath) - const url = await minioClient.presignedGetObject(config.assets.archivesBucket, path.basename(archivePath)) - - return url -} - -module.exports = createArchive diff --git a/src/lib/archives/read.js b/src/lib/archives/read.js deleted file mode 100644 index b8c1b2b..0000000 --- a/src/lib/archives/read.js +++ /dev/null @@ -1,41 +0,0 @@ -const - yauzl = require('yauzl'), - path = require('path') - -const readArchive = function (archivePath) { - const results = {} - const getFile = (entry, zipfile) => { - return new Promise((resolve, reject) => { - let data = '' - zipfile.openReadStream(entry, function (err, readStream) { - if (err) return reject(err) - readStream.on('data', chunk => { - data += chunk.toString() - }) - readStream.on('end', () => resolve(data)) - readStream.on('error', err => reject(err)) - }) - }) - } - return new Promise((resolve, reject) => { - yauzl.open(archivePath, {lazyEntries: true}, async (err, zipfile) => { - if (err) return reject(err) - zipfile.readEntry() - zipfile.on('end', () => resolve(results)) - zipfile.on('error', err => reject(err)) - zipfile.on('entry', async entry => { - if (/\/$/.test(entry.fileName)) zipfile.readEntry() - else { - const type = path.dirname(entry.fileName) - const data = await getFile(entry, zipfile) - const obj = JSON.parse(data) - if (!results[type]) results[type] = [] - results[type].push(obj) - zipfile.readEntry() - } - }) - }) - }) -} - -module.exports = readArchive -- GitLab