Merge pull request #763 from syuilo/#757

#757
This commit is contained in:
こぴなたみぽ 2017-09-06 23:23:13 +09:00 committed by GitHub
commit a94c130140
11 changed files with 514 additions and 0 deletions

1
.gitignore vendored
View File

@ -3,6 +3,7 @@
/node_modules
/built
/uploads
/data
npm-debug.log
*.pem
run.bat

View File

@ -22,6 +22,14 @@ common:
confused: "Confused"
pudding: "Pudding"
post_categories:
music: "Music"
game: "Video Game"
anime: "Anime"
it: "IT"
gadgets: "Gadgets"
photography: "Photography"
input-message-here: "Enter message here"
send: "Send"
delete: "Delete"
@ -80,6 +88,9 @@ common:
mk-post-menu:
pin: "Pin"
pinned: "Pinned"
select: "Select category"
categorize: "Accept"
categorized: "Category reported. Thank you!"
mk-reaction-picker:
choose-reaction: "Pick your reaction"
@ -375,6 +386,7 @@ mobile:
twitter-integration: "Twitter integration"
signin-history: "Sign in history"
api: "API"
link: "MisskeyLink"
settings: "Settings"
signout: "Sign out"

View File

@ -22,6 +22,14 @@ common:
confused: "こまこまのこまり"
pudding: "Pudding"
post_categories:
music: "音楽"
game: "ゲーム"
anime: "アニメ"
it: "IT"
gadgets: "ガジェット"
photography: "写真"
input-message-here: "ここにメッセージを入力"
send: "送信"
delete: "削除"
@ -80,6 +88,9 @@ common:
mk-post-menu:
pin: "ピン留め"
pinned: "ピン留めしました"
select: "カテゴリを選択"
categorize: "決定"
categorized: "カテゴリを報告しました。これによりMisskeyが賢くなり、投稿の自動カテゴライズに役立てられます。ご協力ありがとうございました。"
mk-reaction-picker:
choose-reaction: "リアクションを選択"
@ -375,6 +386,7 @@ mobile:
twitter-integration: "Twitter連携"
signin-history: "ログイン履歴"
api: "API"
link: "Misskeyリンク"
settings: "設定"
signout: "サインアウト"

View File

@ -64,6 +64,7 @@
"@types/webpack": "3.0.10",
"@types/webpack-stream": "3.2.7",
"@types/websocket": "0.0.34",
"@types/msgpack-lite": "^0.1.5",
"chai": "4.1.2",
"chai-http": "3.0.0",
"css-loader": "0.28.7",
@ -120,10 +121,12 @@
"is-root": "1.0.0",
"is-url": "1.2.2",
"js-yaml": "3.9.1",
"mecab-async": "^0.1.0",
"mongodb": "2.2.31",
"monk": "6.0.3",
"morgan": "1.8.2",
"ms": "2.0.0",
"msgpack-lite": "^0.1.26",
"multer": "1.3.0",
"nprogress": "0.2.0",
"os-utils": "0.0.14",

View File

@ -394,6 +394,10 @@ const endpoints: Endpoint[] = [
name: 'posts/trend',
withCredential: true
},
{
name: 'posts/categorize',
withCredential: true
},
{
name: 'posts/reactions',
withCredential: true

View File

@ -0,0 +1,52 @@
/**
* Module dependencies
*/
import $ from 'cafy';
import Post from '../../models/post';
/**
* Categorize a post
*
* @param {any} params
* @param {any} user
* @return {Promise<any>}
*/
module.exports = (params, user) => new Promise(async (res, rej) => {
if (!user.is_pro) {
return rej('This endpoint is available only from a Pro account');
}
// Get 'post_id' parameter
const [postId, postIdErr] = $(params.post_id).id().$;
if (postIdErr) return rej('invalid post_id param');
// Get categorizee
const post = await Post.findOne({
_id: postId
});
if (post === null) {
return rej('post not found');
}
if (post.is_category_verified) {
return rej('This post already has the verified category');
}
// Get 'category' parameter
const [category, categoryErr] = $(params.category).string().or([
'music', 'game', 'anime', 'it', 'gadgets', 'photography'
]).$;
if (categoryErr) return rej('invalid category param');
// Set category
Post.update({ _id: post._id }, {
$set: {
category: category,
is_category_verified: true
}
});
// Send response
res();
});

View File

@ -68,6 +68,9 @@ type Source = {
hook_secret: string;
username: string;
};
categorizer?: {
mecab_command?: string;
};
};
/**

302
src/tools/ai/naive-bayes.js Normal file
View File

@ -0,0 +1,302 @@
// Original source code: https://github.com/ttezel/bayes/blob/master/lib/naive_bayes.js (commit: 2c20d3066e4fc786400aaedcf3e42987e52abe3c)
// CUSTOMIZED BY SYUILO
/*
Expose our naive-bayes generator function
*/
module.exports = function (options) {
return new Naivebayes(options)
}
// keys we use to serialize a classifier's state
var STATE_KEYS = module.exports.STATE_KEYS = [
'categories', 'docCount', 'totalDocuments', 'vocabulary', 'vocabularySize',
'wordCount', 'wordFrequencyCount', 'options'
];
/**
* Initializes a NaiveBayes instance from a JSON state representation.
* Use this with classifier.toJson().
*
* @param {String} jsonStr state representation obtained by classifier.toJson()
* @return {NaiveBayes} Classifier
*/
module.exports.fromJson = function (jsonStr) {
var parsed;
try {
parsed = JSON.parse(jsonStr)
} catch (e) {
throw new Error('Naivebayes.fromJson expects a valid JSON string.')
}
// init a new classifier
var classifier = new Naivebayes(parsed.options)
// override the classifier's state
STATE_KEYS.forEach(function (k) {
if (!parsed[k]) {
throw new Error('Naivebayes.fromJson: JSON string is missing an expected property: `'+k+'`.')
}
classifier[k] = parsed[k]
})
return classifier
}
/**
* Given an input string, tokenize it into an array of word tokens.
* This is the default tokenization function used if user does not provide one in `options`.
*
* @param {String} text
* @return {Array}
*/
var defaultTokenizer = function (text) {
//remove punctuation from text - remove anything that isn't a word char or a space
var rgxPunctuation = /[^(a-zA-ZA-Яa-я0-9_)+\s]/g
var sanitized = text.replace(rgxPunctuation, ' ')
return sanitized.split(/\s+/)
}
/**
* Naive-Bayes Classifier
*
* This is a naive-bayes classifier that uses Laplace Smoothing.
*
* Takes an (optional) options object containing:
* - `tokenizer` => custom tokenization function
*
*/
function Naivebayes (options) {
// set options object
this.options = {}
if (typeof options !== 'undefined') {
if (!options || typeof options !== 'object' || Array.isArray(options)) {
throw TypeError('NaiveBayes got invalid `options`: `' + options + '`. Pass in an object.')
}
this.options = options
}
this.tokenizer = this.options.tokenizer || defaultTokenizer
//initialize our vocabulary and its size
this.vocabulary = {}
this.vocabularySize = 0
//number of documents we have learned from
this.totalDocuments = 0
//document frequency table for each of our categories
//=> for each category, how often were documents mapped to it
this.docCount = {}
//for each category, how many words total were mapped to it
this.wordCount = {}
//word frequency table for each category
//=> for each category, how frequent was a given word mapped to it
this.wordFrequencyCount = {}
//hashmap of our category names
this.categories = {}
}
/**
* Initialize each of our data structure entries for this new category
*
* @param {String} categoryName
*/
Naivebayes.prototype.initializeCategory = function (categoryName) {
if (!this.categories[categoryName]) {
this.docCount[categoryName] = 0
this.wordCount[categoryName] = 0
this.wordFrequencyCount[categoryName] = {}
this.categories[categoryName] = true
}
return this
}
/**
* train our naive-bayes classifier by telling it what `category`
* the `text` corresponds to.
*
* @param {String} text
* @param {String} class
*/
Naivebayes.prototype.learn = function (text, category) {
var self = this
//initialize category data structures if we've never seen this category
self.initializeCategory(category)
//update our count of how many documents mapped to this category
self.docCount[category]++
//update the total number of documents we have learned from
self.totalDocuments++
//normalize the text into a word array
var tokens = self.tokenizer(text)
//get a frequency count for each token in the text
var frequencyTable = self.frequencyTable(tokens)
/*
Update our vocabulary and our word frequency count for this category
*/
Object
.keys(frequencyTable)
.forEach(function (token) {
//add this word to our vocabulary if not already existing
if (!self.vocabulary[token]) {
self.vocabulary[token] = true
self.vocabularySize++
}
var frequencyInText = frequencyTable[token]
//update the frequency information for this word in this category
if (!self.wordFrequencyCount[category][token])
self.wordFrequencyCount[category][token] = frequencyInText
else
self.wordFrequencyCount[category][token] += frequencyInText
//update the count of all words we have seen mapped to this category
self.wordCount[category] += frequencyInText
})
return self
}
/**
* Determine what category `text` belongs to.
*
* @param {String} text
* @return {String} category
*/
Naivebayes.prototype.categorize = function (text) {
var self = this
, maxProbability = -Infinity
, chosenCategory = null
var tokens = self.tokenizer(text)
var frequencyTable = self.frequencyTable(tokens)
//iterate thru our categories to find the one with max probability for this text
Object
.keys(self.categories)
.forEach(function (category) {
//start by calculating the overall probability of this category
//=> out of all documents we've ever looked at, how many were
// mapped to this category
var categoryProbability = self.docCount[category] / self.totalDocuments
//take the log to avoid underflow
var logProbability = Math.log(categoryProbability)
//now determine P( w | c ) for each word `w` in the text
Object
.keys(frequencyTable)
.forEach(function (token) {
var frequencyInText = frequencyTable[token]
var tokenProbability = self.tokenProbability(token, category)
// console.log('token: %s category: `%s` tokenProbability: %d', token, category, tokenProbability)
//determine the log of the P( w | c ) for this word
logProbability += frequencyInText * Math.log(tokenProbability)
})
if (logProbability > maxProbability) {
maxProbability = logProbability
chosenCategory = category
}
})
return chosenCategory
}
/**
* Calculate probability that a `token` belongs to a `category`
*
* @param {String} token
* @param {String} category
* @return {Number} probability
*/
Naivebayes.prototype.tokenProbability = function (token, category) {
//how many times this word has occurred in documents mapped to this category
var wordFrequencyCount = this.wordFrequencyCount[category][token] || 0
//what is the count of all words that have ever been mapped to this category
var wordCount = this.wordCount[category]
//use laplace Add-1 Smoothing equation
return ( wordFrequencyCount + 1 ) / ( wordCount + this.vocabularySize )
}
/**
* Build a frequency hashmap where
* - the keys are the entries in `tokens`
* - the values are the frequency of each entry in `tokens`
*
* @param {Array} tokens Normalized word array
* @return {Object}
*/
Naivebayes.prototype.frequencyTable = function (tokens) {
var frequencyTable = Object.create(null)
tokens.forEach(function (token) {
if (!frequencyTable[token])
frequencyTable[token] = 1
else
frequencyTable[token]++
})
return frequencyTable
}
/**
* Dump the classifier's state as a JSON string.
* @return {String} Representation of the classifier.
*/
Naivebayes.prototype.toJson = function () {
var state = {}
var self = this
STATE_KEYS.forEach(function (k) {
state[k] = self[k]
})
var jsonStr = JSON.stringify(state)
return jsonStr
}
// (original method)
Naivebayes.prototype.export = function () {
var state = {}
var self = this
STATE_KEYS.forEach(function (k) {
state[k] = self[k]
})
return state
}
module.exports.import = function (data) {
var parsed = data
// init a new classifier
var classifier = new Naivebayes()
// override the classifier's state
STATE_KEYS.forEach(function (k) {
if (!parsed[k]) {
throw new Error('Naivebayes.import: data is missing an expected property: `'+k+'`.')
}
classifier[k] = parsed[k]
})
return classifier
}

View File

@ -0,0 +1,57 @@
const bayes = require('./naive-bayes.js');
const MeCab = require('mecab-async');
import Post from '../../api/models/post';
import config from '../../conf';
const classifier = bayes({
tokenizer: this.tokenizer
});
const mecab = new MeCab();
if (config.categorizer.mecab_command) mecab.command = config.categorizer.mecab_command;
// 訓練データ取得
Post.find({
is_category_verified: true
}, {
fields: {
_id: false,
text: true,
category: true
}
}).then(verifiedPosts => {
// 学習
verifiedPosts.forEach(post => {
classifier.learn(post.text, post.category);
});
// 全ての(人間によって証明されていない)投稿を取得
Post.find({
text: {
$exists: true
},
is_category_verified: {
$ne: true
}
}, {
sort: {
_id: -1
},
fields: {
_id: true,
text: true
}
}).then(posts => {
posts.forEach(post => {
console.log(`predicting... ${post._id}`);
const category = classifier.categorize(post.text);
Post.update({ _id: post._id }, {
$set: {
category: category
}
});
});
});
});

View File

@ -0,0 +1,45 @@
import Post from '../../api/models/post';
import User from '../../api/models/user';
export async function predictOne(id) {
console.log(`predict interest of ${id} ...`);
// TODO: repostなども含める
const recentPosts = await Post.find({
user_id: id,
category: {
$exists: true
}
}, {
sort: {
_id: -1
},
limit: 1000,
fields: {
_id: false,
category: true
}
});
const categories = {};
recentPosts.forEach(post => {
if (categories[post.category]) {
categories[post.category]++;
} else {
categories[post.category] = 1;
}
});
}
export async function predictAll() {
const allUsers = await User.find({}, {
fields: {
_id: true
}
});
allUsers.forEach(user => {
predictOne(user._id);
});
}

View File

@ -2,6 +2,18 @@
<div class="backdrop" ref="backdrop" onclick={ close }></div>
<div class="popover { compact: opts.compact }" ref="popover">
<button if={ post.user_id === I.id } onclick={ pin }>%i18n:common.tags.mk-post-menu.pin%</button>
<div if={ I.is_pro && !post.is_category_verified }>
<select ref="categorySelect">
<option value="">%i18n:common.tags.mk-post-menu.select%</option>
<option value="music">%i18n:common.post_categories.music%</option>
<option value="game">%i18n:common.post_categories.game%</option>
<option value="anime">%i18n:common.post_categories.anime%</option>
<option value="it">%i18n:common.post_categories.it%</option>
<option value="gadgets">%i18n:common.post_categories.gadgets%</option>
<option value="photography">%i18n:common.post_categories.photography%</option>
</select>
<button onclick={ categorize }>%i18n:common.tags.mk-post-menu.categorize%</button>
</div>
</div>
<style>
$border-color = rgba(27, 31, 35, 0.15)
@ -111,6 +123,17 @@
});
};
this.categorize = () => {
const category = this.refs.categorySelect.options[this.refs.categorySelect.selectedIndex].value;
this.api('posts/categorize', {
post_id: this.post.id,
category: category
}).then(() => {
if (this.opts.cb) this.opts.cb('categorized', '%i18n:common.tags.mk-post-menu.categorized%');
this.unmount();
});
};
this.close = () => {
this.refs.backdrop.style.pointerEvents = 'none';
anime({