import { readFileSync, writeFileSync } from 'fs'; import axios from 'axios'; type LibraryThingAuthor = { lf: string, fl: string, role: string, } type LibraryThingBook = { books_id: string, title: string, authors: LibraryThingAuthor[], pages: string, lcc: { code: string }, publication: string, publicationDate: string, genre: string[], summary: string, language: string[], } type LibraryThingExport = Record type AuthorMap = Map type AuthorRecord = LibraryThingAuthor & { id: number, updatedAt: string, createdAt: string, } type GenreImport = { name: string, } type GenreRecord = { id: number, name: string, updatedAt: string, createdAt: string, } type BookImport = { books_id: string, title: string, pages?: number, lcc?: string, publication?: string, genre?: { id: number }[], // id of the genre already stored in DB summary?: string, authors?: { id: number }[], // id of the author already stored in DB } type BookRecord = BookImport & { id: number genre: GenreRecord[] } const getAuthToken = async () => { const creds = { email: process.env.EMAIL, password: process.env.PASSWORD, } const loginRequest = await axios.post( `${process.env.BASE_URL}/api/users/login`, creds, ) const authToken = loginRequest.data.token return authToken } const makeAuthorKey = (author: LibraryThingAuthor) => { return `${author.role}: ${author.lf}` } const parseAuthors = (libraryThingData: LibraryThingExport): AuthorMap => { const authors = new Map() as AuthorMap const bookIds = Object.keys(libraryThingData) bookIds.forEach(bookId => { const currentBook = libraryThingData[bookId] currentBook.authors.forEach(a => { const authorKey = makeAuthorKey(a) const existingAuthor = authors.get(authorKey) if (!existingAuthor) authors.set(authorKey, a) }) }); return authors } const importAuthors = async (authors: AuthorMap, authToken: string) => { const headers = { 'Authorization': authToken, 'Content-Type': 'application/json', } const allAuthorsInDb = await axios.get(`${process.env.BASE_URL}/api/authors?limit=10000`, { headers }) const alreadyAddedAuthors = (allAuthorsInDb?.data?.docs || []) as AuthorRecord[] const importPromises = [...authors.values()].map(async (a) => { const foundAuthor = alreadyAddedAuthors.find(addedAuthor => { if (addedAuthor.lf !== a.lf) return false if (!a.role && !addedAuthor.role) return true if (a.role && a.role === addedAuthor.role) return true return false }) if (foundAuthor) { //console.log('Author already exists - ', makeAuthorKey(foundAuthor)) return } return await axios.post( `${process.env.BASE_URL}/api/authors`, a, { headers, withCredentials: true } ).then(saveResponse => { const savedAuthor = saveResponse?.data?.doc as AuthorRecord || null if (savedAuthor?.id) { console.log('Added Author:', savedAuthor.lf) alreadyAddedAuthors.push(savedAuthor) } }) }); await Promise.allSettled(importPromises) return alreadyAddedAuthors } const parseGenre = (libraryThingData: LibraryThingExport) => { const parsedGenres: string[] = [] const bookIds = Object.keys(libraryThingData) bookIds.forEach(bookId => { const currentBook = libraryThingData[bookId] currentBook.genre?.forEach(g => { if (!parsedGenres.includes(g)) parsedGenres.push(g) }) }); return parsedGenres } const importGenres = async (genres: string[], authToken: string) => { const headers = { 'Authorization': authToken, 'Content-Type': 'application/json', } const allGenresInDb = await axios.get(`${process.env.BASE_URL}/api/genre?limit=10000`, { headers }) const alreadyAddedGenres = (allGenresInDb?.data?.docs || []) as GenreRecord[] const importGenrePromises = genres.map(async g => { const foundGenre = alreadyAddedGenres.find(added => added.name === g) if (foundGenre) new Promise(() => { }) return axios.post( `${process.env.BASE_URL}/api/genre`, { name: g }, { headers }, ).then(res => { const importedGenre = res?.data?.doc || null if (importedGenre) alreadyAddedGenres.push(importedGenre) }) }); await Promise.allSettled(importGenrePromises) return alreadyAddedGenres } const parseBooks = (libraryThingData: LibraryThingExport, genreRecords: GenreRecord[], authorRecords: AuthorRecord[]) => { const books: BookImport[] = [] const bookIds = Object.keys(libraryThingData) bookIds.forEach(bookId => { const curr = libraryThingData[bookId] const parsedBook: BookImport = { books_id: curr.books_id, title: curr.title, lcc: curr.lcc?.code || '', publication: curr.publication, summary: curr.summary, genre: [] as { id: number }[], authors: [] as { id: number }[] } const formatedPages = parseInt(curr.pages?.trim() || '', 10) if (formatedPages) parsedBook.pages = formatedPages curr.genre?.forEach(g => { const matchingGenreRecord = genreRecords.find(rec => rec.name === g) if (matchingGenreRecord) parsedBook.genre?.push({ id: matchingGenreRecord.id }) }) curr.authors.forEach(a => { const matchingAuthor = authorRecords.find(match => { if (match.lf !== a.lf) return false if (!a.role && !match.role) return true if (a.role && a.role === match.role) return true return false }) if (matchingAuthor) parsedBook.authors?.push({ id: matchingAuthor.id }) }) books.push(parsedBook) }); return books } const importBooks = async (books: BookImport[], authToken: string) => { const headers = { 'Authorization': authToken, 'Content-Type': 'application/json', } const allBooksInDb = await axios.get(`${process.env.BASE_URL}/api/books?limit=10000`, { headers }) const alreadyAddedBooks = (allBooksInDb?.data?.docs || []) as BookRecord[] books.forEach(async b => { const foundExistingBook = alreadyAddedBooks.find(added => added.books_id === b.books_id) if (foundExistingBook) return const importBookResponse = await axios.post(`${process.env.BASE_URL}/api/books`, b, { headers }) const importedBook = importBookResponse?.data?.doc || null console.log('Added book: ', importedBook) }); } const main = async () => { const libraryThingData = JSON.parse(readFileSync('./libraryThingExport.json', 'utf8')) as LibraryThingExport const authToken = await getAuthToken() const authors = parseAuthors(libraryThingData) const authorRecords = await importAuthors(authors, authToken) const genres = parseGenre(libraryThingData) const genreRecords = await importGenres(genres, authToken) const books = parseBooks(libraryThingData, genreRecords, authorRecords) importBooks(books, authToken) } main()