Merge pull request #1386 from akihikodaki/uri

Enforce URI uniquness
This commit is contained in:
syuilo 2018-04-04 00:12:40 +09:00 committed by GitHub
commit 9f659271b5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 82 additions and 48 deletions

View File

@ -10,7 +10,7 @@ import * as debug from 'debug';
import fileType = require('file-type'); import fileType = require('file-type');
import prominence = require('prominence'); import prominence = require('prominence');
import DriveFile, { getGridFSBucket } from '../models/drive-file'; import DriveFile, { IMetadata, getGridFSBucket } from '../models/drive-file';
import DriveFolder from '../models/drive-folder'; import DriveFolder from '../models/drive-folder';
import { pack } from '../models/drive-file'; import { pack } from '../models/drive-file';
import event, { publishDriveStream } from '../publishers/stream'; import event, { publishDriveStream } from '../publishers/stream';
@ -45,7 +45,8 @@ const addFile = async (
name: string = null, name: string = null,
comment: string = null, comment: string = null,
folderId: mongodb.ObjectID = null, folderId: mongodb.ObjectID = null,
force: boolean = false force: boolean = false,
uri: string = null
) => { ) => {
log(`registering ${name} (user: ${getAcct(user)}, path: ${path})`); log(`registering ${name} (user: ${getAcct(user)}, path: ${path})`);
@ -224,12 +225,18 @@ const addFile = async (
properties['avgColor'] = averageColor; properties['avgColor'] = averageColor;
} }
return addToGridFS(detectedName, readable, mime, { const metadata = {
userId: user._id, userId: user._id,
folderId: folder !== null ? folder._id : null, folderId: folder !== null ? folder._id : null,
comment: comment, comment: comment,
properties: properties properties: properties
}); } as IMetadata;
if (uri !== null) {
metadata.uri = uri;
}
return addToGridFS(detectedName, readable, mime, metadata);
}; };
/** /**

View File

@ -8,7 +8,7 @@ import * as request from 'request';
const log = debug('misskey:common:drive:upload_from_url'); const log = debug('misskey:common:drive:upload_from_url');
export default async (url, user, folderId = null): Promise<IDriveFile> => { export default async (url, user, folderId = null, uri = null): Promise<IDriveFile> => {
let name = URL.parse(url).pathname.split('/').pop(); let name = URL.parse(url).pathname.split('/').pop();
if (!validateFileName(name)) { if (!validateFileName(name)) {
name = null; name = null;
@ -35,7 +35,7 @@ export default async (url, user, folderId = null): Promise<IDriveFile> => {
.on('error', rej); .on('error', rej);
}); });
const driveFile = await create(user, path, name, null, folderId); const driveFile = await create(user, path, name, null, folderId, false, uri);
// clean-up // clean-up
fs.unlink(path, (e) => { fs.unlink(path, (e) => {

View File

@ -6,6 +6,8 @@ import monkDb, { nativeDbConn } from '../db/mongodb';
const DriveFile = monkDb.get<IDriveFile>('driveFiles.files'); const DriveFile = monkDb.get<IDriveFile>('driveFiles.files');
DriveFile.createIndex('metadata.uri', { sparse: true, unique: true });
export default DriveFile; export default DriveFile;
const getGridFSBucket = async (): Promise<mongodb.GridFSBucket> => { const getGridFSBucket = async (): Promise<mongodb.GridFSBucket> => {
@ -18,17 +20,21 @@ const getGridFSBucket = async (): Promise<mongodb.GridFSBucket> => {
export { getGridFSBucket }; export { getGridFSBucket };
export type IMetadata = {
properties: any;
userId: mongodb.ObjectID;
folderId: mongodb.ObjectID;
comment: string;
uri: string;
};
export type IDriveFile = { export type IDriveFile = {
_id: mongodb.ObjectID; _id: mongodb.ObjectID;
uploadDate: Date; uploadDate: Date;
md5: string; md5: string;
filename: string; filename: string;
contentType: string; contentType: string;
metadata: { metadata: IMetadata;
properties: any;
userId: mongodb.ObjectID;
folderId: mongodb.ObjectID;
}
}; };
export function validateFileName(name: string): boolean { export function validateFileName(name: string): boolean {

View File

@ -11,6 +11,8 @@ import { pack as packFile } from './drive-file';
const Post = db.get<IPost>('posts'); const Post = db.get<IPost>('posts');
Post.createIndex('uri', { sparse: true, unique: true });
export default Post; export default Post;
export function isValidText(text: string): boolean { export function isValidText(text: string): boolean {
@ -49,6 +51,7 @@ export type IPost = {
heading: number; heading: number;
speed: number; speed: number;
}; };
uri: string;
}; };
/** /**

View File

@ -1,15 +0,0 @@
import * as mongodb from 'mongodb';
import db from '../db/mongodb';
const RemoteUserObject = db.get<IRemoteUserObject>('remoteUserObjects');
export default RemoteUserObject;
export type IRemoteUserObject = {
_id: mongodb.ObjectID;
uri: string;
object: {
$ref: string;
$id: mongodb.ObjectID;
}
};

View File

@ -1,6 +1,8 @@
import { JSDOM } from 'jsdom'; import { JSDOM } from 'jsdom';
import { ObjectID } from 'mongodb';
import config from '../../config'; import config from '../../config';
import RemoteUserObject, { IRemoteUserObject } from '../../models/remote-user-object'; import DriveFile from '../../models/drive-file';
import Post from '../../models/post';
import { IRemoteUser } from '../../models/user'; import { IRemoteUser } from '../../models/user';
import uploadFromUrl from '../../drive/upload-from-url'; import uploadFromUrl from '../../drive/upload-from-url';
import createPost from '../../post/create'; import createPost from '../../post/create';
@ -8,15 +10,13 @@ import distributePost from '../../post/distribute';
import Resolver from './resolver'; import Resolver from './resolver';
const createDOMPurify = require('dompurify'); const createDOMPurify = require('dompurify');
function createRemoteUserObject($ref, $id, { id }) { type IResult = {
const object = { $ref, $id }; resolver: Resolver;
object: {
if (!id) { $ref: string;
return { object }; $id: ObjectID;
} };
};
return RemoteUserObject.insert({ uri: id, object });
}
class Creator { class Creator {
private actor: IRemoteUser; private actor: IRemoteUser;
@ -27,17 +27,23 @@ class Creator {
this.distribute = distribute; this.distribute = distribute;
} }
private async createImage(image) { private async createImage(resolver: Resolver, image) {
if ('attributedTo' in image && this.actor.account.uri !== image.attributedTo) { if ('attributedTo' in image && this.actor.account.uri !== image.attributedTo) {
throw new Error(); throw new Error();
} }
const { _id } = await uploadFromUrl(image.url, this.actor); const { _id } = await uploadFromUrl(image.url, this.actor, image.id || null);
return createRemoteUserObject('driveFiles.files', _id, image); return {
resolver,
object: { $ref: 'driveFiles.files', $id: _id }
};
} }
private async createNote(resolver: Resolver, note) { private async createNote(resolver: Resolver, note) {
if ('attributedTo' in note && this.actor.account.uri !== note.attributedTo) { if (
('attributedTo' in note && this.actor.account.uri !== note.attributedTo) ||
typeof note.id !== 'string'
) {
throw new Error(); throw new Error();
} }
@ -61,10 +67,10 @@ class Creator {
userId: this.actor._id, userId: this.actor._id,
appId: null, appId: null,
viaMobile: false, viaMobile: false,
geo: undefined geo: undefined,
uri: note.id
}, null, null, []); }, null, null, []);
const promisedRemoteUserObject = createRemoteUserObject('posts', inserted._id, note);
const promises = []; const promises = [];
if (this.distribute) { if (this.distribute) {
@ -89,18 +95,45 @@ class Creator {
await Promise.all(promises); await Promise.all(promises);
return promisedRemoteUserObject; return {
resolver,
object: { $ref: 'posts', id: inserted._id }
};
} }
public async create(parentResolver: Resolver, value): Promise<Array<Promise<IRemoteUserObject>>> { public async create(parentResolver: Resolver, value): Promise<Array<Promise<IResult>>> {
const collection = await parentResolver.resolveCollection(value); const collection = await parentResolver.resolveCollection(value);
return collection.object.map(async element => { return collection.object.map(async element => {
if (typeof element === 'string') { if (typeof element === 'string') {
const object = RemoteUserObject.findOne({ uri: element }); try {
await Promise.all([
DriveFile.findOne({ 'metadata.uri': element }).then(file => {
if (file === null) {
return;
}
if (object !== null) { throw {
return object; $ref: 'driveFile.files',
$id: file._id
};
}, () => {}),
Post.findOne({ uri: element }).then(post => {
if (post === null) {
return;
}
throw {
$ref: 'posts',
$id: post._id
};
}, () => {})
]);
} catch (object) {
return {
resolver: collection.resolver,
object
};
} }
} }
@ -108,7 +141,7 @@ class Creator {
switch (object.type) { switch (object.type) {
case 'Image': case 'Image':
return this.createImage(object); return this.createImage(resolver, object);
case 'Note': case 'Note':
return this.createNote(resolver, object); return this.createNote(resolver, object);