import { readFileSync, writeFileSync } from 'fs'; import axios from 'axios'; type LibraryThingAuthor = { lf: string, fl: string, role: string, } type LibraryThingBook = { books_id: string, title: string, authors: LibraryThingAuthor[], pages: string, lcc: { code: string }, publication: string, publicationDate: string, genre: string[], summary: string, language: string[], originalisbn: string, asin: string } type LibraryThingExport = Record type AuthorMap = Map type AuthorRecord = LibraryThingAuthor & { id: number, updatedAt: string, createdAt: string, } type GenreImport = { name: string, } type GenreRecord = { id: number, name: string, updatedAt: string, createdAt: string, } type BookImport = { books_id: string, title: string, pages?: number, lcc?: string, publication?: string, genre?: { id: number }[], // id of the genre already stored in DB summary?: string, authors?: { id: number }[], // id of the author already stored in DB isbn?: string, asin?: string, } type BookRecord = BookImport & { id: number genre: GenreRecord[] } type CopyRecord = { id: number, condition: number, book: { id: number }, repository: { id: number }, } const getAuthToken = async () => { const creds = { email: process.env.EMAIL, password: process.env.PASSWORD, } const loginRequest = await axios.post( `${process.env.BASE_URL}/api/users/login`, creds, ) const authToken = loginRequest.data.token return authToken } const makeAuthorKey = (author: LibraryThingAuthor) => { return `${author.role}: ${author.lf}` } const parseAuthors = (libraryThingData: LibraryThingExport): AuthorMap => { const authors = new Map() as AuthorMap const bookIds = Object.keys(libraryThingData) bookIds.forEach(bookId => { const currentBook = libraryThingData[bookId] currentBook.authors.forEach(a => { const authorKey = makeAuthorKey(a) const existingAuthor = authors.get(authorKey) if (!existingAuthor) authors.set(authorKey, a) }) }); return authors } const importAuthors = async (authors: AuthorMap, authToken: string) => { const headers = { 'Authorization': authToken, 'Content-Type': 'application/json', } const allAuthorsInDb = await axios.get(`${process.env.BASE_URL}/api/authors?limit=10000`, { headers }) const alreadyAddedAuthors = (allAuthorsInDb?.data?.docs || []) as AuthorRecord[] const importPromises = [...authors.values()].map(async (a) => { const foundAuthor = alreadyAddedAuthors.find(addedAuthor => { if (addedAuthor.lf !== a.lf) return false if (!a.role && !addedAuthor.role) return true if (a.role && a.role === addedAuthor.role) return true return false }) if (foundAuthor) { //console.log('Author already exists - ', makeAuthorKey(foundAuthor)) return } return await axios.post( `${process.env.BASE_URL}/apallBooksInDbi/authors`, a, { headers, withCredentials: true } ).then(saveResponse => { const savedAuthor = saveResponse?.data?.doc as AuthorRecord || null if (savedAuthor?.id) { console.log('Added Author:', savedAuthor.lf) alreadyAddedAuthors.push(savedAuthor) } }) }); await Promise.allSettled(importPromises) return alreadyAddedAuthors } const parseGenre = (libraryThingData: LibraryThingExport) => { const parsedGenres: string[] = [] const bookIds = Object.keys(libraryThingData) bookIds.forEach(bookId => { const currentBook = libraryThingData[bookId] currentBook.genre?.forEach(g => { if (!parsedGenres.includes(g)) parsedGenres.push(g) }) }); return parsedGenres } const importGenres = async (genres: string[], authToken: string) => { const headers = { 'Authorization': authToken, 'Content-Type': 'application/json', } const allGenresInDb = await axios.get(`${process.env.BASE_URL}/api/genre?limit=10000`, { headers }) const alreadyAddedGenres = (allGenresInDb?.data?.docs || []) as GenreRecord[] const importGenrePromises = genres.map(async g => { const foundGenre = alreadyAddedGenres.find(added => added.name === g) if (foundGenre) new Promise(() => { }) return axios.post( `${process.env.BASE_URL}/api/genre`, { name: g }, { headers }, ).then(res => { const importedGenre = res?.data?.doc || null if (importedGenre) alreadyAddedGenres.push(importedGenre) }) }); await Promise.allSettled(importGenrePromises) return alreadyAddedGenres } const parseBooks = (libraryThingData: LibraryThingExport, genreRecords: GenreRecord[], authorRecords: AuthorRecord[]) => { const books: BookImport[] = [] const bookIds = Object.keys(libraryThingData) bookIds.forEach(bookId => { const curr = libraryThingData[bookId] const parsedBook: BookImport = { books_id: curr.books_id, title: curr.title, lcc: curr.lcc?.code || '', publication: curr.publication, summary: curr.summary, genre: [] as { id: number }[], authors: [] as { id: number }[], isbn: curr.originalisbn, asin: curr.asin, } const formatedPages = parseInt(curr.pages?.trim() || '', 10) if (formatedPages) parsedBook.pages = formatedPages curr.genre?.forEach(g => { const matchingGenreRecord = genreRecords.find(rec => rec.name === g) if (matchingGenreRecord) parsedBook.genre?.push({ id: matchingGenreRecord.id }) }) curr.authors.forEach(a => { const matchingAuthor = authorRecords.find(match => { if (match.lf !== a.lf) return false if (!a.role && !match.role) return true if (a.role && a.role === match.role) return true return false }) if (matchingAuthor) parsedBook.authors?.push({ id: matchingAuthor.id }) }) books.push(parsedBook) }); return books } const importBooks = async (books: BookImport[], authToken: string, options?: { updateFields: (keyof BookRecord)[] }) => { const headers = { 'Authorization': authToken, 'Content-Type': 'application/json', } const shouldUpdate = !!options?.updateFields?.length const allBooksInDb = await axios.get(`${process.env.BASE_URL}/api/books?limit=10000`, { headers }) const alreadyAddedBooks = (allBooksInDb?.data?.docs || []) as BookRecord[] for (let i = 0; i < books.length; i++) { const foundExistingBook = alreadyAddedBooks.find(added => added.books_id === books[i].books_id) if (foundExistingBook && shouldUpdate) { let newBookProps: Partial = {} options.updateFields.forEach(key => { (newBookProps as any)[key as keyof BookRecord] = (books[i] as any)[key] // This is stupid, why cant TS just knoe that keyof is valid for square brakets? }); console.log('Updating book: ', foundExistingBook.id, newBookProps) const updateBookResponse = await axios.patch(`${process.env.BASE_URL}/api/books/${foundExistingBook.id}`, newBookProps, { headers }) const updatededBook = updateBookResponse?.data?.doc || null console.log('Updated book: ', updatededBook) continue } else if (foundExistingBook && !shouldUpdate) continue const importBookResponse = await axios.post(`${process.env.BASE_URL}/api/books`, books[i], { headers }) const importedBook = importBookResponse?.data?.doc || null console.log('Added book: ', importedBook) } } const copyAllExistingDbBooksIntoRepo = async (repoId: number, authToken: string) => { const headers = { 'Authorization': authToken, 'Content-Type': 'application/json', } const allBooksInDbResponse = await axios.get(`${process.env.BASE_URL}/api/books?limit=10000`, { headers }) const allBooksInDb = (allBooksInDbResponse?.data?.docs || []) as BookRecord[] const allCopiesInDb = await axios.get(`${process.env.BASE_URL}/api/copies?limit=10000`, { headers }) const alreadyAddedCopies = (allCopiesInDb?.data?.docs || []) as CopyRecord[] for (let i = 0; i < allBooksInDb.length; i++) { const foundExistingCopy = alreadyAddedCopies.find(c => { return c.book.id === allBooksInDb[i].id }) if (foundExistingCopy) continue const copyProps: Partial = { book: { id: allBooksInDb[i].id }, repository: { id: repoId }, } const createCopyResponse = await axios.post(`${process.env.BASE_URL}/api/copies`, copyProps, { headers }) const importedCopy = createCopyResponse?.data?.doc || null console.log('Added copy: ', importedCopy.id) } } const main = async () => { console.log("Reading LIbray Thing JSON") const libraryThingData = JSON.parse(readFileSync('./libraryThingExport.json', 'utf8')) as LibraryThingExport console.log("Logging in...") const authToken = await getAuthToken() console.log("Parsing Authors...") const authors = parseAuthors(libraryThingData) console.log("Importing Authors...") const authorRecords = await importAuthors(authors, authToken) console.log(`Parsed ${authorRecords.length} Authors`) console.log("Parsing Genres...") const genres = parseGenre(libraryThingData) console.log("Importing Genres...") const genreRecords = await importGenres(genres, authToken) console.log(`Parsed ${genreRecords.length} Genres`) console.log("Parsing Books...") const books = parseBooks(libraryThingData, genreRecords, authorRecords) console.log("Importing Books...") await importBooks(books, authToken) //importBooks(books, authToken, { // updateFields: ['asin', 'isbn'] //}) console.log("Imported/Updated books") console.log("Copying Books into Repo...") await copyAllExistingDbBooksIntoRepo(1, authToken) console.log("Copied Books") } main()