tristonarmstrong/XeNote
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finished constructing backlink from internal links

Browse Source 
TODO: performance improvement, and code reability.
This commit is contained in:
Tuan Cao 2022-04-07 16:52:41 +07:00
parent b47d1053d8
commit 2bb96504c9
5 changed files with 170 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
components/MDContent.js
View File

@ -12,7 +12,7 @@ function BackLinks({linkList}) {
?
linkList.map(aLink =>
<li>
<Link href={aLink.link}>
<Link href={aLink.slug}>
<a>
{aLink.title}
</a>

10
lib/_post.js
View File

@ -7,8 +7,6 @@ import { Node } from "./node"
var remark = require('remark')
const unified = require('unified')
const markdown = require('remark-parse')
const { wikiLinkPlugin } = require('remark-wiki-link');
var guide = require('remark-preset-lint-markdown-style-guide')
var html = require('remark-html')
var report = require('vfile-reporter')
@ -43,10 +41,10 @@ export function getSortedPostsData() {
// let processor = unified()
// .use(markdown, { gfm: true })
// .use(wikiLinkPlugin)
const htmlContent = Remark.getHtmlContent(fileContent, {
fileNames:fileNames,
permalink: `/note/${permalink}`
})
// const htmlContent = Remark.getHtmlContent(fileContent, {
// fileNames:fileNames,
// permalink: `/note/${permalink}`
// })
//unified()
//.use(markdown)

103
lib/transformer.js
View File

@ -8,6 +8,7 @@ import html from "remark-html";
import frontmatter from "remark-frontmatter";
import externalLinks from "remark-external-links";
import highlight from "remark-highlight.js";
import {Node} from "./node";
const postsDirectory = path.join(process.cwd(), 'posts')
const isFile = fileName => {
@ -18,7 +19,7 @@ export const Transformer = {
haveFrontMatter: function (content) {
//console.log("\t Front matter data content", content)
if (!content) return false
var indexOfFirst = content.indexOf("---")
const indexOfFirst = content.indexOf("---");
//console.log("\t Front matter data firstIndex ", indexOfFirst)
//console.log("index first", indexOfFirst)
if (indexOfFirst === -1) {
@ -37,7 +38,6 @@ export const Transformer = {
getHtmlContent: function (content, {fileNames}) {
let htmlContent = []
let internalLinks = []
const sanitizedContent = Transformer.preprocessThreeDashes(content)
unified()
@ -46,61 +46,36 @@ export const Transformer = {
.use(externalLinks, {target: "_blank", rel: ['noopener']})
.use(frontmatter, ['yaml', 'toml'])
.use(wikiLinkPlugin, {
permalinks:fileNames,
pageResolver: function(pageName){
const name = [Transformer.parseFileNameFromPath(pageName)]
//console.log("\n\nwiki internal links", Transformer.parseFileNameFromPath(name[0]));
const backLink = {
title: name,
link: Transformer.parseFileNameFromPath(name[0]),
shortSummary: name
}
internalLinks.push(backLink);
return name
permalinks: null,
pageResolver: function (pageName) {
return [Transformer.parseFileNameFromPath(pageName)]
},
hrefTemplate: function(permalink){
hrefTemplate: function (permalink) {
permalink = Transformer.normalizeFileName(permalink)
permalink = permalink.replace("ç","c").replace("ı","i").replace("ş","s")
//console.log("wiki pemalink", permalink);
permalink = permalink.replace("ç", "c").replace("ı", "i").replace("ş", "s")
return `/note/${permalink}`
},
aliasDivider:"|"
aliasDivider: "|"
})
.use(html)
.process(sanitizedContent,
function (err, file) {
//console.log("asd", String(file).slice(0,50))
//console.error("remark: ", report(err || file))
htmlContent.push(String(file).replace("\n", ""))
}
htmlContent.push(String(file).replace("\n", ""))
}
)
htmlContent = htmlContent.join("")
htmlContent = htmlContent.join("")
htmlContent = htmlContent.split("---")
return [htmlContent, internalLinks]
return [htmlContent]
},
// getReact : function (content, {fileNames}) {
// const sanitizedContent = Transformer.preprocessThreeDashes(content)
// let result = null
// Transformer.getProcessor(content, {fileNames})
// .use(rehypeReact, {createElement, Fragment})
// .process(sanitizedContent, file => {
// result = file.result
// })
//
// return result
// },
/* SANITIZE MARKDOWN FOR --- */
preprocessThreeDashes: function (content) {
var indexOfFirst = content.indexOf("---")
const indexOfFirst = content.indexOf("---");
if (indexOfFirst === -1) {
return content
}
var indexOfSecond = content.indexOf("---", (indexOfFirst + 1))
const indexOfSecond = content.indexOf("---", (indexOfFirst + 1));
const frontPart = content.slice(0, indexOfSecond);
const contentPart = content.slice(indexOfSecond);
const processedContent = contentPart.split("---").join("")
@ -111,7 +86,7 @@ export const Transformer = {
/* Normalize File Names */
normalizeFileName: function (filename) {
var processedFileName = filename.replace(".md", "")
let processedFileName = filename.replace(".md", "");
processedFileName = processedFileName.replace('(', '').replace(')', '')
processedFileName = processedFileName.split(" ").join("-")
// processedFileName = processedFileName.toLowerCase()
@ -136,12 +111,12 @@ export const Transformer = {
},
/* Pair provided and existing Filenames*/
pairCurrentFile: function (provided, ListOfFilePaths) {
pairCurrentFile: function (provided, listOfFilePaths) {
//console.log(provided, ListOfFilePaths)
const providedSanitizedFileName = Transformer.normalizeFileName(provided);
// Map file paths and return true if it pairs with provided
const possibleFilePath = ListOfFilePaths.filter(possibleFilePath => {
const possibleFilePath = listOfFilePaths.filter(possibleFilePath => {
const possibleFileName = Transformer.parseFileNameFromPath(possibleFilePath);
const possibleSanitizedFileName = Transformer.normalizeFileName(possibleFileName)
//console.log("----", providedSanitizedFileName, possibleSanitizedFileName)
@ -152,5 +127,49 @@ export const Transformer = {
})
console.log("p---", possibleFilePath)
return possibleFilePath[0]
},
getInternalLinks: function (aFilePath) {
const filePaths = Node.getFiles(postsDirectory);
const currentFilePath = Transformer.pairCurrentFile(aFilePath, filePaths)
const fileContent = Node.readFileSync(aFilePath);
const internalLinks = []
const sanitizedContent = Transformer.preprocessThreeDashes(fileContent)
unified()
.use(markdown, {gfm: true})
.use(wikiLinkPlugin, {
pageResolver: function (pageName) {
// let name = [Transformer.parseFileNameFromPath(pageName)];
let slug = Transformer.parseFileNameFromPath(pageName);
if (slug.includes('#')) {
console.log(slug)
}
const canonicalSlug = slug.includes('#') ? slug.split('#')[0] : slug
const backLink = {
title: canonicalSlug,
slug: canonicalSlug,
shortSummary: canonicalSlug
}
if (canonicalSlug != null && internalLinks.find(aLink => aLink.slug === canonicalSlug ) == null) {
internalLinks.push(backLink);
}
return [canonicalSlug]
}
,
hrefTemplate: function (permalink) {
permalink = Transformer.normalizeFileName(permalink)
permalink = permalink.replace("ç", "c").replace("ı", "i").replace("ş", "s")
return `/note/${permalink}`
},
aliasDivider: "|"
})
.use(html)
.processSync(sanitizedContent)
return internalLinks;
}
}

213
lib/utils.js
View File

@ -9,135 +9,109 @@ export function getAllFileNames() {
return Node.getFiles(postsDirectory).map(f => Transformer.parseFileNameFromPath(f))
}
export function getSinglePost(filename) {
console.log("\n\nFile is scanning: ", filename)
// List of filenames that will provide existing links to wikilink
var filePaths = Node.getFiles(postsDirectory)
export function getFileNames(filename) {
let filePaths = Node.getFiles(postsDirectory);
const fileNames = filePaths.map(f => Transformer.parseFileNameFromPath(f))
//console.log("\t filenames: ",fileNames, "\n")
// IF filename is not sidebar.md THEN Exclude sidebar.md from file list
var currentFilePath;
if (filename === "sidebar"){
// IF filename is not sidebar.md THEN Exclude sidebar.md from file list
let currentFilePath;
if (filename === "sidebar") {
//console.log(111)
currentFilePath = path.join(postsDirectory, "/sidebar.md")
}
else if (filename === "index"){
} else if (filename === "index") {
//console.log(222)
currentFilePath = path.join(postsDirectory, "/index.md")
}
else {
//console.log(333)
} else {
//TODO remove reference to index/sidebar md
filePaths = filePaths.filter(f => !(f.endsWith("sidebar.md") && f.endsWith("index.md")))
//console.log("\tDirectory is scanning to find corresponding filename")
currentFilePath = Transformer.pairCurrentFile(filename, filePaths)
//console.log("\tScan is finished. Founded filepath", currentFilePath, "\n")
}
return {fileNames, currentFilePath};
}
export function getSinglePost(filename) {
console.log("\n\nFile is scanning: ", filename)
// List of filenames that will provide existing links to wikilink
let {fileNames, currentFilePath} = getFileNames(filename);
//console.log("currentFilePath: ", currentFilePath)
var fileContent = Node.readFileSync(currentFilePath)
const currentFileFrontMatter = Transformer.getFrontMatterData(fileContent)
//console.log("\tFounded front matter data: ", currentFileFrontMatter, "\n")
const currentFileFrontMatter = Transformer.getFrontMatterData(fileContent)
//console.log("\tFounded front matter data: ", currentFileFrontMatter, "\n")
// fileContent = Transformer.preprocessThreeDashes(fileContent)
//fileContent = fileContent.split("---").join("")
//console.log("filecontent end")
//fileContent = fileContent.split("---").join("")
//console.log("filecontent end")
const [htmlContent, backlinks] = Transformer.getHtmlContent(fileContent, {
fileNames:fileNames,
})
//console.log("hrmlcontents and backlinks")
return {
id:filename,
...currentFileFrontMatter,
data:htmlContent,
backLinks: backlinks
}
}
const [htmlContent] = Transformer.getHtmlContent(fileContent, {
fileNames: fileNames,
})
//console.log("hrmlcontents and backlinks")
return {
id: filename,
...currentFileFrontMatter,
data: htmlContent,
}
export function getAllBacklinks(){
//console.log("\n\nBacklinks are scanning")
//var bimap = new BiMap
var internalLinks = []
// Get file names under /posts
const filePaths = Node.getFiles(postsDirectory).filter(f => !f.endsWith("sidebar.md"))
const fileNames = filePaths.map(f => Transformer.parseFileNameFromPath(f))
//console.log("\tFounded filePaths: ", fileNames)
var allBacklinkData = filePaths.map(fileName => {
// Remove ".md" from file name to get id
const slug = Transformer.parseFileNameFromPath(fileName)
//console.log("filename", fileNames)
const fileData = {
id:slug
}
//console.log("AllBacklinks slug", slug)
// Read markdown file as string
var fileContent = Node.readFileSync(fileName, 'utf8')
const frontmatterData = Transformer.getFrontMatterData(fileContent)
const requiredParameters = ["title", "description"]
requiredParameters.forEach(param => {
if (frontmatterData[param])
fileData[param] = frontmatterData[param]
})
//fileContent = fileContent.split("---").join("")
const [htmlContent, backlinks] = Transformer.getHtmlContent(fileContent, {
fileNames:fileNames,
})
// Check if scanned slug post has any internal links
const existingInternalLink = backlinks.filter(bl => fileNames.includes(bl))
fileData.to = existingInternalLink
fileData.href = slug === "index" ? "/" : `/note/${slug}`
//console.log("\n\nbacklinks",[ slug, [backlinks]] )
//bimap.push(slug, backlinks)
// Check if internal link exists
//const internalLinks = backlinks.filter(bl => fileNames.includes(bl))
internalLinks.push(fileData)
//console.log("bimap: ", bimap.key(slug))
// Combine the data with the slug
//return backlinkList.length > 0 ? JSON.stringify(backlinkList) : null
})
//console.log("founded internal links for ", internalLinks)
//console.log("\n\ninternal list: ", internalLinks)
return internalLinks
//return [allBacklinkData.filter(bl => bl !== null), JSON.stringify(fileNames)]
}
export function getGraphData(){
const backlinkData = getAllBacklinks()
export function constructBackLinks() {
const elements = []
// First create Nodes
backlinkData.forEach(el => {
const node = {data: {id: el.id}};
const filePaths = getContentPaths()
const edges = []
const nodes = []
if(el.title){
node.data.title = el.title
}
if (el.description){
node.data.description = el.description
}
elements.push(node)
filePaths.forEach( filename => {
const {currentFilePath, fileNames} = getFileNames(filename)
const internalLinks = Transformer.getInternalLinks(currentFilePath)
internalLinks.forEach(aLink => {
if (aLink.slug === null || aLink.slug.length === 0) return
const anEdge = {
source: filename,
target: aLink.slug,
}
edges.push(anEdge)
if (nodes.findIndex(aNode => aNode.slug === aLink.slug) === -1) {
nodes.push(aLink)
}
})
}
)
return {nodes, edges};
}
export function getGraphData() {
const backlinkData = constructBackLinks()
const elements = []
// First create Nodes
backlinkData.forEach(el => {
const node = {data: {id: el.id}};
if (el.title) {
node.data.title = el.title
}
if (el.description) {
node.data.description = el.description
}
elements.push(node)
}
)
// Second create Edges
backlinkData.forEach(el => {
// check if has any internal link
if (el.to.length > 0){
if (el.to.length > 0) {
// create edge from element to its links
el.to.forEach(linkElement => {
const edge = {
@ -151,36 +125,35 @@ export function getGraphData(){
})
}
})
return elements
}
export function getPostListData() {
//console.log("\n\nAll Posts are scanning")
// Get file names under /posts
const filePaths = Node.getFiles(postsDirectory).filter(f => !(f.endsWith("index") || f.endsWith("sidebar")))
const fileNames = filePaths.map(f => Transformer.parseFileNameFromPath(f))
//console.log("filePaths", filePaths)
return fileNames
export function getContentPaths() {
//console.log("\n\nAll Posts are scanning")
// Get file names under /posts
const filePaths = Node.getFiles(postsDirectory).filter(f => !(f.endsWith("index") || f.endsWith("sidebar")))
return filePaths.map(f => Transformer.parseFileNameFromPath(f))
}
export function getDirectoryData() {
const filteredDirectory = dirTree(postsDirectory,{ extensions: /\.md/ });
const convertedData = convertObject(filteredDirectory)
// console.log()
// const array = getFlattenArray(convertedData)
return convertedData
const filteredDirectory = dirTree(postsDirectory, {extensions: /\.md/});
return convertObject(filteredDirectory)
}
let _counter = 0;
export function convertObject(thisObject) {
const children = []
let routerPath = getPostListData().find(fileName => fileName === Transformer.normalizeFileName(thisObject.name) ) || null
routerPath = routerPath ? '/note/' +routerPath : null
const newObject = {name: thisObject.name, children: children, id: (_counter++).toString(), routePath: routerPath || null };
let routerPath = getContentPaths().find(fileName => fileName === Transformer.normalizeFileName(thisObject.name)) || null
routerPath = routerPath ? '/note/' + routerPath : null
const newObject = {
name: thisObject.name,
children: children,
id: (_counter++).toString(),
routePath: routerPath || null
};
if (thisObject.children != null && thisObject.children.length > 0) {
thisObject.children.forEach(aChild => {
@ -204,6 +177,6 @@ function flat(array) {
return result;
}
export function getFlattenArray (thisObject) {
export function getFlattenArray(thisObject) {
return flat(thisObject.children)
}

17
pages/note/[id].js
View File

@ -1,12 +1,11 @@
import Head from "next/head";
import Layout from "../../components/layout";
import {
getPostListData,
getContentPaths,
getSinglePost,
getGraphData,
convertObject,
getDirectoryData,
getAllFileNames
constructBackLinks, getFileNames
} from "../../lib/utils";
import FolderTree from "../../components/FolderTree";
import {getFlattenArray} from "../../lib/utils";
@ -29,7 +28,7 @@ export default function Home({note, backLinks, fileNames, tree, flattenNodes}) {
}
export async function getStaticPaths() {
const allPostsData = getPostListData();
const allPostsData = getContentPaths();
const paths = allPostsData.map(p => ({params: {id: p}}))
return {
paths,
@ -37,11 +36,17 @@ export async function getStaticPaths() {
};
}
const {nodes, edges} = constructBackLinks()
export function getStaticProps({params}) {
const note = getSinglePost(params.id);
const tree = convertObject(getDirectoryData());
const flattenNodes = getFlattenArray(tree)
const fileNames = getAllFileNames()
// const fileNames = getAllFileNames()
const { fileNames} = getFileNames(params.id)
const listOfEdges = edges.filter(anEdge => anEdge.target === params.id)
const internalLinks = listOfEdges.map(anEdge => nodes.find(aNode => aNode.slug === anEdge.source)).filter(element => element !== undefined)
return {
props: {
@ -49,7 +54,7 @@ export function getStaticProps({params}) {
tree: tree,
flattenNodes: flattenNodes,
fileNames: fileNames,
backLinks: note.backLinks
backLinks: internalLinks
},
};
}