Skip to content

Commit

Permalink
fix: agirls routes (#12778)
Browse files Browse the repository at this point in the history 
* fix: agirls routes

* refactor: combine duplicated code

---------
  • Loading branch information
@henry40408
henry40408 committed Jul 11, 2023
1 parent cc7ea6a commit d513d6b
Show file tree
Hide file tree
Showing 5 changed files with 81 additions and 100 deletions.
56 changes: 14 additions & 42 deletions lib/v2/agirls/index.js
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,15 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { parseDate } = require('@/utils/parse-date');
const timezone = require('@/utils/timezone');
const path = require('path');
const { art } = require('@/utils/render');
const { CookieJar } = require('tough-cookie');

const cookieJar = new CookieJar();
const { baseUrl, parseArticle } = require('./utils');

module.exports = async (ctx) => {
const baseUrl = 'https://agirls.aotter.net';
const category = ctx.params.category ?? '';
const cookieJar = new CookieJar();
const { category = '' } = ctx.params;
const link = `${baseUrl}/posts${category ? `/${category}` : ''}`;

const response = await got({
url: baseUrl + '/posts' + (category ? `/${category}` : ''),
url: link,
headers: {
Referer: baseUrl,
},
Expand All @@ -22,53 +18,29 @@ module.exports = async (ctx) => {

const $ = cheerio.load(response.data);

let items = $('div a.flex.flex-col')
.map((_, item) => {
let items = $('.ag-post-item__link')
.toArray()
.map((item) => {
item = $(item);
return {
title: item.find('h2').text().trim(),
link: baseUrl + item.attr('href'),
title: item.text().trim(),
link: `${baseUrl}${item.attr('href')}`,
};
})
.get();

items = await Promise.all(
items.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
url: item.link,
headers: {
Referer: baseUrl + '/posts/' + (category ? `/${category}` : ''),
},
cookieJar,
});
const content = cheerio.load(detailResponse.data);

item.category = content('a.mr-2.bg-neutral-lightest.px-2.py-1.text-sm.text-neutral-light')
.toArray()
.map((e) => content(e).text().trim().replace('#', ''));
item.description = art(path.join(__dirname, 'templates/description.art'), {
desc: content('div.post-content').html(),
});
item.pubDate = timezone(parseDate(content('time').text().trim()), +8);
item.author = content('a.mr-1.text-primary').text().trim();
});

return item;
})
)
);
items = await Promise.all(items.map((item) => ctx.cache.tryGet(item.link, () => parseArticle(item, link, cookieJar))));

ctx.state.data = {
title: $('head title').text().trim(),
link: baseUrl + '/' + category,
link,
description: $('head meta[name=description]').attr('content'),
item: items,
language: $('html').attr('lang'),
};

ctx.state.json = {
title: $('head title').text().trim(),
link: baseUrl + '/' + category,
link,
description: $('head meta[name=description]').attr('content'),
item: items,
language: $('html').attr('lang'),
Expand Down
1 change: 0 additions & 1 deletion lib/v2/agirls/templates/description.art
View file

This file was deleted.

59 changes: 16 additions & 43 deletions lib/v2/agirls/topic.js
View file
Original file line number Diff line number Diff line change
@@ -1,70 +1,43 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { parseDate } = require('@/utils/parse-date');
const timezone = require('@/utils/timezone');
const path = require('path');
const { art } = require('@/utils/render');
const { baseUrl, parseArticle } = require('./utils');

module.exports = async (ctx) => {
const baseUrl = 'https://agirls.aotter.net';
const topic = ctx.params.topic;

const { topic } = ctx.params;
const link = `${baseUrl}/topic/${topic}`;
const response = await got({
url: baseUrl + '/topic/' + topic,
url: link,
headers: {
Referer: baseUrl,
},
});

const $ = cheerio.load(response.data);

let items = $('div.col-12 div.row a')
.map((_, item) => {
const ldJson = JSON.parse($('script[type="application/ld+json"]').text());
let items = $('.ag-post-item__link')
.toArray()
.map((item) => {
item = $(item);
return {
title: item.find('h3').text().trim(),
link: baseUrl + item.attr('href'),
title: item.text().trim(),
link: `${baseUrl}${item.attr('href')}`,
};
})
.get();

items = await Promise.all(
items.map((item) =>
ctx.cache.tryGet(item.link, async () => {
const detailResponse = await got({
url: item.link,
headers: {
Referer: baseUrl + '/topic/' + topic,
},
});
const content = cheerio.load(detailResponse.data);

item.category = content('a.mr-2.bg-neutral-lightest.px-2.py-1.text-sm.text-neutral-light')
.toArray()
.map((e) => content(e).text().trim().replace('#', ''));
item.description = art(path.join(__dirname, 'templates/description.art'), {
desc: content('div.post-content').html(),
});
item.pubDate = timezone(parseDate(content('time').text().trim()), +8);
item.author = content('a.mr-1.text-primary').text().trim();
});

return item;
})
)
);
items = await Promise.all(items.map((item) => ctx.cache.tryGet(item.link, () => parseArticle(item, link, undefined))));

ctx.state.data = {
title: $('head title').text().trim(),
link: baseUrl + '/' + topic,
description: $('head meta[name=description]').attr('content'),
link,
description: ldJson['@graph'][0].description,
item: items,
language: $('html').attr('lang'),
};

ctx.state.json = {
title: $('head title').text().trim(),
link: baseUrl + '/' + topic,
description: $('head meta[name=description]').attr('content'),
link,
description: ldJson['@graph'][0].description,
item: items,
language: $('html').attr('lang'),
};
Expand Down
28 changes: 14 additions & 14 deletions lib/v2/agirls/topic_list.js
View file
Original file line number Diff line number Diff line change
@@ -1,42 +1,42 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { baseUrl } = require('./utils');

module.exports = async (ctx) => {
const baseUrl = 'https://agirls.aotter.net';
const category = 'topic';
const link = `${baseUrl}/${category}`;

const response = await got({
url: baseUrl + '/' + category,
url: `${baseUrl}/${category}`,
headers: {
Referer: baseUrl,
},
});

const $ = cheerio.load(response.data);

const items = $('div div.border-b.border-neutral-lighter')
.map((_, item) => {
const items = $('.ag-topic')
.toArray()
.map((item) => {
item = $(item);
const link = baseUrl + item.find('a').attr('href');
return ctx.cache.tryGet(link, () => ({
title: item.find('h2').text().trim(),
description: item.find('a').attr('href').replace('/topic/', ''),
link,
}));
})
.get();
return {
title: item.find('.ag-topic__link').text().trim(),
description: item.find('.ag-topic__summery').text().trim(),
link: `${baseUrl}${item.find('.ag-topic__link').attr('href')}`,
};
});

ctx.state.data = {
title: $('head title').text().trim(),
link: baseUrl + '/' + category,
link,
description: $('head meta[name=description]').attr('content'),
item: items,
language: $('html').attr('lang'),
};

ctx.state.json = {
title: $('head title').text().trim(),
link: baseUrl + '/' + category,
link,
description: $('head meta[name=description]').attr('content'),
item: items,
language: $('html').attr('lang'),
Expand Down
37 changes: 37 additions & 0 deletions lib/v2/agirls/utils.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
const got = require('@/utils/got');
const cheerio = require('cheerio');
const { parseDate } = require('@/utils/parse-date');

const baseUrl = 'https://agirls.aotter.net';

const parseArticle = async (item, referer, cookieJar) => {
const detailResponse = await got({
url: item.link,
headers: {
Referer: referer,
},
cookieJar,
});
const content = cheerio.load(detailResponse.data);

item.category = [
...new Set(
content('.ag-article__tag')
.toArray()
.map((e) => content(e).text().trim().replace('#', ''))
),
];
const ldJson = JSON.parse(content('script[type="application/ld+json"]').text());

item.description = content('.ag-article__content').html();
item.pubDate = parseDate(ldJson['@graph'][0].datePublished); // 2023-07-05T12:11:36+08:00
item.updated = parseDate(ldJson['@graph'][0].dateModified); // 2023-07-05T12:11:36+08:00
item.author = ldJson['@graph'][0].author.map((a) => a.name).join(', ');

return item;
};

module.exports = {
baseUrl,
parseArticle,
};

0 comments on commit d513d6b

Please sign in to comment.