Skip to content

Commit

Permalink
update
Browse files Browse the repository at this point in the history
  • Loading branch information
@Gosev
Gosev committed Nov 19, 2023
1 parent 5a54a29 commit 2e530f5
Show file tree
Hide file tree
Showing 5 changed files with 97 additions and 52 deletions.
42 changes: 42 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,44 @@
# notion-parse
An NPM module for downloading and Notion content and saving it as Markdown for NextJS

I use this to download my Notion content and save it as Markdown for my NextJS blog. It's a work in progress, but it's working for me so far.

I also use ContentLayer to make sure my fontmatter has the right fields.

## Usage

Here is how I use the module in my NextJS project.

```js

// @ts-check

const NotionParse = require('@kodaps/notion-parse');
const dotenv = require('dotenv');

dotenv.config();

const go = async () => {

if (process.env.NOTION_TOKEN) {
await NotionParse.parseNotion(process.env.NOTION_TOKEN, './src/content', [
{ databaseId: process.env.NOTION_PORTFOLIO_DATABASE_ID || '', contentType: 'Portfolio' },
{ databaseId: process.env.NOTION_NEWSLETTER_DATABASE_ID || '', contentType: 'Newsletter', languageField: 'lang' },
//{ databaseId: process.env.NOTION_PAGE_DATABASE_ID || '', contentType: 'Page', languageField: 'lang' },
{ databaseId: process.env.NOTION_POST_DATABASE_ID || '', contentType: 'Post', languageField: 'lang', filterFields: [ 'translation', 'createdAt', 'status', 'Type'] },
{ databaseId: process.env.NOTION_BITS_DATABASE_ID || '', contentType: 'Bit', languageField: 'lang' },
])
}

};

go().then(() => {
console.log('Done');
});

```

This supposed several things :
1. that the files are stored in a subfolder of the folder passed in as parameter (here `./src/content`) based on the content type
2. that the ContentLayer type names map to the subfolders. So for instance for the `Post` content type, the files will be stored in `./src/content/post`
3. that the Notion token and database IDs are stored in environment variables, and that there is one database per content type
4 changes: 2 additions & 2 deletions lib/fileManagement.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
Object.defineProperty(exports, "__esModule", { value: true });
exports.getImageFolderPath = exports.getFilePath = exports.getImageFolder = exports.getFileFolder = void 0;
var getFileFolder = function (filetype, lang) {
var langBit = lang ? "".concat(lang, "/") : '';
var langBit = (!!lang) ? "".concat(lang, "/") : '';
var contentFolder = filetype.toLowerCase();
return "./src/content/".concat(contentFolder, "/").concat(langBit);
};
Expand All @@ -14,7 +14,7 @@ var getImageFolder = function (filetype) {
exports.getImageFolder = getImageFolder;
var getFilePath = function (slug, filetype, lang) {
var fileFolder = (0, exports.getFileFolder)(filetype, lang);
return "".concat(fileFolder, "/").concat(slug, ".md");
return "".concat(fileFolder).concat(slug, ".md");
};
exports.getFilePath = getFilePath;
var getImageFolderPath = function (slug, filetype) {
Expand Down
11 changes: 4 additions & 7 deletions lib/index.d.ts
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,12 @@
import { Client } from "@notionhq/client";
import { DatabaseObjectResponse, PageObjectResponse, PartialDatabaseObjectResponse, PartialPageObjectResponse } from "@notionhq/client/build/src/api-endpoints";
interface DocumentType {
databaseId: string;
languageField?: string;
contentType: string;
filterFields?: Array<string>;
}
declare const parseNotionPage: (page: PageObjectResponse | PartialPageObjectResponse | PartialDatabaseObjectResponse | DatabaseObjectResponse, contentType: string) => Promise<{
export declare const parseNotionPage: (page: PageObjectResponse | PartialPageObjectResponse | PartialDatabaseObjectResponse | DatabaseObjectResponse, contentType: string) => Promise<{
[key: string]: any;
}>;
declare const getDatabase: (notion: Client, database_id: string, contentType: string) => Promise<{
[key: string]: any;
}[]>;
declare const parseNotion: (token: string, contentRoot: string, contentTypes: Array<DocumentType>) => Promise<void>;
export { getDatabase, parseNotionPage, parseNotion, };
export declare const parseNotion: (token: string, contentRoot: string, contentTypes: Array<DocumentType>) => Promise<void>;
export {};
90 changes: 48 additions & 42 deletions lib/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,17 +36,16 @@ var __generator = (this && this.__generator) || function (thisArg, body) {
}
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.parseNotion = exports.parseNotionPage = exports.getDatabase = void 0;
exports.parseNotion = exports.parseNotionPage = void 0;
var client_1 = require("@notionhq/client");
var notion_to_md_1 = require("notion-to-md");
var fileManagement_1 = require("./fileManagement");
//const slugify = require('slugify');
var yaml = require('yaml');
var fs = require('fs');
var http = require('https');
var slugify = require('slugify');
// const dotenv = require('dotenv');
var Jimp = require('jimp');
var sleep = function (ms) { return new Promise(function (resolve) { return setTimeout(resolve, ms); }); };
var notionClient = null;
var n2m = null;
var setNotionSecret = function (auth) {
Expand All @@ -66,9 +65,7 @@ var downloadImage = function (fileUrl, destination) { return __awaiter(void 0, v
case 0:
src = destination;
file = './public' + src;
console.log('Checking if file exists: ', file);
if (!!fs.existsSync(file)) return [3 /*break*/, 2];
console.log("Downloading image ".concat(file));
return [4 /*yield*/, wget(fileUrl, file)];
case 1:
_a.sent(); //element.properties.image.files[0].file.url, file);
Expand All @@ -93,7 +90,6 @@ var manageImage = function (properties, url, contentType, name) { return __await
case 0: return [4 /*yield*/, getFieldInfo(properties, 'title', contentType)];
case 1:
title = _a.sent();
console.log('Managing image for ', title, url);
return [4 /*yield*/, getFieldInfo(properties, 'slug', contentType)];
case 2:
slug = (_a.sent()) || slugify(title, {
Expand All @@ -104,20 +100,18 @@ var manageImage = function (properties, url, contentType, name) { return __await
throw new Error('No slug');
}
destination = (0, fileManagement_1.getImageFolder)(contentType) + name;
console.log("Destination: ", destination);
return [4 /*yield*/, downloadImage(url, destination)];
case 3: return [2 /*return*/, _a.sent()];
}
});
}); };
var getFieldInfo = function (properties, name, contentType) { return __awaiter(void 0, void 0, void 0, function () {
var element, type, _a, url;
var _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m;
return __generator(this, function (_o) {
switch (_o.label) {
var _b, _c, _d, _e, _f, _g, _h, _j;
return __generator(this, function (_k) {
switch (_k.label) {
case 0:
element = properties[name];
//console.log('Getting field ', name, element);
if (!element) {
return [2 /*return*/, null];
}
Expand All @@ -131,27 +125,38 @@ var getFieldInfo = function (properties, name, contentType) { return __awaiter(v
case 'checkbox': return [3 /*break*/, 5];
case 'number': return [3 /*break*/, 6];
case 'select': return [3 /*break*/, 7];
case 'multi_select': return [3 /*break*/, 8];
case 'files': return [3 /*break*/, 9];
case 'created_time': return [3 /*break*/, 8];
case 'last_edited_time': return [3 /*break*/, 9];
case 'email': return [3 /*break*/, 10];
case 'status': return [3 /*break*/, 11];
case 'phone_number': return [3 /*break*/, 12];
case 'relation': return [3 /*break*/, 13];
case 'multi_select': return [3 /*break*/, 14];
case 'files': return [3 /*break*/, 15];
}
return [3 /*break*/, 11];
return [3 /*break*/, 17];
case 1: return [2 /*return*/, (_b = element.title[0]) === null || _b === void 0 ? void 0 : _b.plain_text];
case 2: return [2 /*return*/, (_c = element.rich_text[0]) === null || _c === void 0 ? void 0 : _c.plain_text];
case 3: return [2 /*return*/, element.date.start];
case 3: return [2 /*return*/, (_d = element.date) === null || _d === void 0 ? void 0 : _d.start];
case 4: return [2 /*return*/, element.url];
case 5: return [2 /*return*/, element.checkbox];
case 6: return [2 /*return*/, element.number];
case 7: return [2 /*return*/, (_d = element.select) === null || _d === void 0 ? void 0 : _d.name];
case 8: return [2 /*return*/, element.multi_select.map(function (item) { return item.name; })];
case 9:
console.log((_e = element.files[0]) === null || _e === void 0 ? void 0 : _e.file, (_f = element.files[0]) === null || _f === void 0 ? void 0 : _f.url, (_g = element.files[0]) === null || _g === void 0 ? void 0 : _g.type, (_h = element.files[0]) === null || _h === void 0 ? void 0 : _h.name);
url = ((_j = element.files[0]) === null || _j === void 0 ? void 0 : _j.url) || ((_l = (_k = element.files[0]) === null || _k === void 0 ? void 0 : _k.file) === null || _l === void 0 ? void 0 : _l.url);
case 7: return [2 /*return*/, (_e = element.select) === null || _e === void 0 ? void 0 : _e.name];
case 8: return [2 /*return*/, element.created_time];
case 9: return [2 /*return*/, element.last_edited_time];
case 10: return [2 /*return*/, element.email];
case 11: return [2 /*return*/, element.status];
case 12: return [2 /*return*/, element.phone_number];
case 13: return [2 /*return*/, element.relation.map(function (item) { return item.id; })];
case 14: return [2 /*return*/, element.multi_select.map(function (item) { return item.name; })];
case 15:
url = ((_f = element.files[0]) === null || _f === void 0 ? void 0 : _f.url) || ((_h = (_g = element.files[0]) === null || _g === void 0 ? void 0 : _g.file) === null || _h === void 0 ? void 0 : _h.url);
if (!url) {
return [2 /*return*/, null];
}
return [4 /*yield*/, manageImage(properties, url, contentType, (_m = element.files[0]) === null || _m === void 0 ? void 0 : _m.name)];
case 10: return [2 /*return*/, _o.sent()];
case 11: throw new Error("Unknown type ".concat(type));
return [4 /*yield*/, manageImage(properties, url, contentType, (_j = element.files[0]) === null || _j === void 0 ? void 0 : _j.name)];
case 16: return [2 /*return*/, _k.sent()];
case 17: throw new Error("Unknown type ".concat(type));
}
});
}); };
Expand All @@ -165,11 +170,10 @@ function wget(url, dest) {
wget(String(response.headers.location), dest);
}
else {
console.log('Dpwnloading', url, 'to', dest);
console.log('Downloading', url, 'to', dest);
var file_1 = fs.createWriteStream(dest);
response.pipe(file_1);
file_1.on('finish', function () {
console.log('Download done');
file_1.close();
res();
});
Expand Down Expand Up @@ -215,7 +219,7 @@ var parseNotionPage = function (page, contentType) { return __awaiter(void 0, vo
exports.parseNotionPage = parseNotionPage;
var checkFolder = function (dir) {
if (!fs.existsSync(dir)) {
fs.mkdirSync(dir);
fs.mkdirSync(dir, { recursive: true });
}
};
var getDatabase = function (notion, database_id, contentType) { return __awaiter(void 0, void 0, void 0, function () {
Expand All @@ -234,8 +238,7 @@ var getDatabase = function (notion, database_id, contentType) { return __awaiter
case 2:
if (!(_i < results_1.length)) return [3 /*break*/, 5];
page = results_1[_i];
console.log('page: ', page);
return [4 /*yield*/, parseNotionPage(page, contentType)];
return [4 /*yield*/, (0, exports.parseNotionPage)(page, contentType)];
case 3:
item = _a.sent();
ret.push(item);
Expand All @@ -247,7 +250,6 @@ var getDatabase = function (notion, database_id, contentType) { return __awaiter
}
});
}); };
exports.getDatabase = getDatabase;
var saveFile = function (frontMatter, type, languageField) { return __awaiter(void 0, void 0, void 0, function () {
var notionId, lang, title, slug, mdblocks, imageBlocks, images, imagePath, _i, imageBlocks_1, block, data, url, name_1, filename, src, file, mdBody, _a, images_1, image, newFile;
return __generator(this, function (_b) {
Expand All @@ -259,14 +261,14 @@ var saveFile = function (frontMatter, type, languageField) { return __awaiter(vo
notionId = frontMatter['notionId'];
lang = languageField ? frontMatter[languageField] : '';
title = frontMatter['title'];
console.log('Processing page: ', title);
if (!title && !frontMatter['slug']) {
throw new Error("No title or slug in front matter for ".concat(notionId, " of type ").concat(type));
}
slug = frontMatter['slug'] || slugify(title, {
lower: true,
strict: true,
});
frontMatter['slug'] = slug;
return [4 /*yield*/, n2m.pageToMarkdown(notionId)];
case 1:
mdblocks = _b.sent();
Expand Down Expand Up @@ -311,35 +313,32 @@ var saveFile = function (frontMatter, type, languageField) { return __awaiter(vo
mdBody.parent = mdBody.parent.replace(image.url, image.src);
}
newFile = (0, fileManagement_1.getFilePath)(slug, type, lang);
console.log('preparing to save: ', newFile);
try {
fs.writeFileSync(newFile, toFrontMatter(frontMatter) + mdBody.parent);
}
catch (e) {
console.log('error title: ', title);
console.log('error with file: ', newFile);
console.error(e);
}
return [2 /*return*/];
}
});
}); };
var parseNotion = function (token, contentRoot, contentTypes) { return __awaiter(void 0, void 0, void 0, function () {
var _i, contentTypes_1, type, databaseId, lang, contentType, database, _a, database_1, page;
return __generator(this, function (_b) {
switch (_b.label) {
var _i, contentTypes_1, type, databaseId, lang, contentType, database, _a, database_1, page, _b, _c, field;
return __generator(this, function (_d) {
switch (_d.label) {
case 0:
setNotionSecret(token);
addDocumentTypes(contentTypes);
console.log('Parsing notion content: ', contentRoot, contentTypes);
if (!notionClient) {
throw new Error('Notion client incorretly setup');
}
_i = 0, contentTypes_1 = contentTypes;
_b.label = 1;
_d.label = 1;
case 1:
if (!(_i < contentTypes_1.length)) return [3 /*break*/, 7];
type = contentTypes_1[_i];
console.log('Processing type: ', type);
databaseId = type.databaseId;
lang = type.languageField;
contentType = type.contentType || databaseId;
Expand All @@ -351,16 +350,23 @@ var parseNotion = function (token, contentRoot, contentTypes) { return __awaiter
}
return [4 /*yield*/, getDatabase(notionClient, databaseId, contentType)];
case 2:
database = _b.sent();
database = _d.sent();
_a = 0, database_1 = database;
_b.label = 3;
_d.label = 3;
case 3:
if (!(_a < database_1.length)) return [3 /*break*/, 6];
page = database_1[_a];
sleep(400);
for (_b = 0, _c = (type.filterFields || []); _b < _c.length; _b++) {
field = _c[_b];
if (page[field]) {
delete page[field];
}
}
return [4 /*yield*/, saveFile(page, contentType, lang)];
case 4:
_b.sent();
_b.label = 5;
_d.sent();
_d.label = 5;
case 5:
_a++;
return [3 /*break*/, 3];
Expand Down
2 changes: 1 addition & 1 deletion src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ export const parseNotionPage = async (page:PageObjectResponse| PartialPageObject

const checkFolder = (dir: string) => {
if (!fs.existsSync(dir)){
fs.mkdirSync(dir);
fs.mkdirSync(dir, { recursive: true });
}
}

Expand Down

0 comments on commit 2e530f5

Please sign in to comment.