337 lines
9.6 KiB
TypeScript
337 lines
9.6 KiB
TypeScript
import { readFileSync, writeFileSync } from 'fs';
|
|
import axios from 'axios';
|
|
|
|
type LibraryThingAuthor = {
|
|
lf: string,
|
|
fl: string,
|
|
role: string,
|
|
}
|
|
|
|
type LibraryThingBook = {
|
|
books_id: string,
|
|
title: string,
|
|
authors: LibraryThingAuthor[],
|
|
pages: string,
|
|
lcc: {
|
|
code: string
|
|
},
|
|
publication: string,
|
|
publicationDate: string,
|
|
genre: string[],
|
|
summary: string,
|
|
language: string[],
|
|
originalisbn: string,
|
|
asin: string
|
|
}
|
|
|
|
type LibraryThingExport = Record<string, LibraryThingBook>
|
|
|
|
type AuthorMap = Map<string, LibraryThingAuthor>
|
|
|
|
type AuthorRecord = LibraryThingAuthor & {
|
|
id: number,
|
|
updatedAt: string,
|
|
createdAt: string,
|
|
}
|
|
|
|
type GenreImport = {
|
|
name: string,
|
|
}
|
|
|
|
type GenreRecord = {
|
|
id: number,
|
|
name: string,
|
|
updatedAt: string,
|
|
createdAt: string,
|
|
}
|
|
|
|
type BookImport = {
|
|
books_id: string,
|
|
title: string,
|
|
pages?: number,
|
|
lcc?: string,
|
|
publication?: string,
|
|
genre?: { id: number }[], // id of the genre already stored in DB
|
|
summary?: string,
|
|
authors?: { id: number }[], // id of the author already stored in DB
|
|
isbn?: string,
|
|
asin?: string,
|
|
}
|
|
|
|
type BookRecord = BookImport & {
|
|
id: number
|
|
genre: GenreRecord[]
|
|
}
|
|
|
|
type CopyRecord = {
|
|
id: number,
|
|
condition: number,
|
|
book: { id: number },
|
|
repository: { id: number },
|
|
}
|
|
|
|
|
|
const getAuthToken = async () => {
|
|
const creds = {
|
|
email: process.env.EMAIL,
|
|
password: process.env.PASSWORD,
|
|
}
|
|
|
|
const loginRequest = await axios.post(
|
|
`${process.env.BASE_URL}/api/users/login`,
|
|
creds,
|
|
)
|
|
|
|
const authToken = loginRequest.data.token
|
|
return authToken
|
|
}
|
|
|
|
|
|
const makeAuthorKey = (author: LibraryThingAuthor) => {
|
|
return `${author.role}: ${author.lf}`
|
|
}
|
|
|
|
const parseAuthors = (libraryThingData: LibraryThingExport): AuthorMap => {
|
|
const authors = new Map<string, LibraryThingAuthor>() as AuthorMap
|
|
|
|
const bookIds = Object.keys(libraryThingData)
|
|
bookIds.forEach(bookId => {
|
|
const currentBook = libraryThingData[bookId]
|
|
currentBook.authors.forEach(a => {
|
|
const authorKey = makeAuthorKey(a)
|
|
const existingAuthor = authors.get(authorKey)
|
|
if (!existingAuthor) authors.set(authorKey, a)
|
|
})
|
|
});
|
|
|
|
return authors
|
|
}
|
|
|
|
const importAuthors = async (authors: AuthorMap, authToken: string) => {
|
|
const headers = {
|
|
'Authorization': authToken,
|
|
'Content-Type': 'application/json',
|
|
}
|
|
|
|
const allAuthorsInDb = await axios.get(`${process.env.BASE_URL}/api/authors?limit=10000`, { headers })
|
|
|
|
const alreadyAddedAuthors = (allAuthorsInDb?.data?.docs || []) as AuthorRecord[]
|
|
|
|
const importPromises = [...authors.values()].map(async (a) => {
|
|
const foundAuthor = alreadyAddedAuthors.find(addedAuthor => {
|
|
if (addedAuthor.lf !== a.lf) return false
|
|
|
|
if (!a.role && !addedAuthor.role) return true
|
|
|
|
if (a.role && a.role === addedAuthor.role) return true
|
|
|
|
return false
|
|
})
|
|
|
|
if (foundAuthor) {
|
|
//console.log('Author already exists - ', makeAuthorKey(foundAuthor))
|
|
return
|
|
}
|
|
|
|
|
|
return await axios.post(
|
|
`${process.env.BASE_URL}/apallBooksInDbi/authors`,
|
|
a, { headers, withCredentials: true }
|
|
).then(saveResponse => {
|
|
const savedAuthor = saveResponse?.data?.doc as AuthorRecord || null
|
|
if (savedAuthor?.id) {
|
|
console.log('Added Author:', savedAuthor.lf)
|
|
alreadyAddedAuthors.push(savedAuthor)
|
|
}
|
|
})
|
|
});
|
|
|
|
await Promise.allSettled(importPromises)
|
|
return alreadyAddedAuthors
|
|
}
|
|
|
|
const parseGenre = (libraryThingData: LibraryThingExport) => {
|
|
const parsedGenres: string[] = []
|
|
|
|
const bookIds = Object.keys(libraryThingData)
|
|
bookIds.forEach(bookId => {
|
|
const currentBook = libraryThingData[bookId]
|
|
currentBook.genre?.forEach(g => {
|
|
if (!parsedGenres.includes(g)) parsedGenres.push(g)
|
|
})
|
|
});
|
|
|
|
return parsedGenres
|
|
}
|
|
|
|
const importGenres = async (genres: string[], authToken: string) => {
|
|
const headers = {
|
|
'Authorization': authToken,
|
|
'Content-Type': 'application/json',
|
|
}
|
|
|
|
const allGenresInDb = await axios.get(`${process.env.BASE_URL}/api/genre?limit=10000`, { headers })
|
|
|
|
const alreadyAddedGenres = (allGenresInDb?.data?.docs || []) as GenreRecord[]
|
|
|
|
const importGenrePromises = genres.map(async g => {
|
|
const foundGenre = alreadyAddedGenres.find(added => added.name === g)
|
|
if (foundGenre) new Promise(() => { })
|
|
|
|
return axios.post(
|
|
`${process.env.BASE_URL}/api/genre`,
|
|
{ name: g },
|
|
{ headers },
|
|
).then(res => {
|
|
const importedGenre = res?.data?.doc || null
|
|
if (importedGenre) alreadyAddedGenres.push(importedGenre)
|
|
})
|
|
});
|
|
|
|
await Promise.allSettled(importGenrePromises)
|
|
return alreadyAddedGenres
|
|
}
|
|
|
|
const parseBooks = (libraryThingData: LibraryThingExport, genreRecords: GenreRecord[], authorRecords: AuthorRecord[]) => {
|
|
const books: BookImport[] = []
|
|
|
|
const bookIds = Object.keys(libraryThingData)
|
|
bookIds.forEach(bookId => {
|
|
const curr = libraryThingData[bookId]
|
|
|
|
const parsedBook: BookImport = {
|
|
books_id: curr.books_id,
|
|
title: curr.title,
|
|
lcc: curr.lcc?.code || '',
|
|
publication: curr.publication,
|
|
summary: curr.summary,
|
|
genre: [] as { id: number }[],
|
|
authors: [] as { id: number }[],
|
|
isbn: curr.originalisbn,
|
|
asin: curr.asin,
|
|
}
|
|
|
|
const formatedPages = parseInt(curr.pages?.trim() || '', 10)
|
|
if (formatedPages) parsedBook.pages = formatedPages
|
|
|
|
curr.genre?.forEach(g => {
|
|
const matchingGenreRecord = genreRecords.find(rec => rec.name === g)
|
|
if (matchingGenreRecord) parsedBook.genre?.push({ id: matchingGenreRecord.id })
|
|
})
|
|
|
|
curr.authors.forEach(a => {
|
|
const matchingAuthor = authorRecords.find(match => {
|
|
if (match.lf !== a.lf) return false
|
|
if (!a.role && !match.role) return true
|
|
if (a.role && a.role === match.role) return true
|
|
return false
|
|
})
|
|
|
|
if (matchingAuthor) parsedBook.authors?.push({ id: matchingAuthor.id })
|
|
})
|
|
|
|
books.push(parsedBook)
|
|
});
|
|
|
|
return books
|
|
}
|
|
|
|
const importBooks = async (books: BookImport[], authToken: string, options?: { updateFields: (keyof BookRecord)[] }) => {
|
|
const headers = {
|
|
'Authorization': authToken,
|
|
'Content-Type': 'application/json',
|
|
}
|
|
|
|
const shouldUpdate = !!options?.updateFields?.length
|
|
|
|
const allBooksInDb = await axios.get(`${process.env.BASE_URL}/api/books?limit=10000`, { headers })
|
|
const alreadyAddedBooks = (allBooksInDb?.data?.docs || []) as BookRecord[]
|
|
|
|
for (let i = 0; i < books.length; i++) {
|
|
const foundExistingBook = alreadyAddedBooks.find(added => added.books_id === books[i].books_id)
|
|
if (foundExistingBook && shouldUpdate) {
|
|
|
|
let newBookProps: Partial<BookRecord> = {}
|
|
options.updateFields.forEach(key => {
|
|
(newBookProps as any)[key as keyof BookRecord] = (books[i] as any)[key] // This is stupid, why cant TS just knoe that keyof is valid for square brakets?
|
|
});
|
|
|
|
console.log('Updating book: ', foundExistingBook.id, newBookProps)
|
|
const updateBookResponse = await axios.patch(`${process.env.BASE_URL}/api/books/${foundExistingBook.id}`, newBookProps, { headers })
|
|
const updatededBook = updateBookResponse?.data?.doc || null
|
|
console.log('Updated book: ', updatededBook)
|
|
continue
|
|
} else if (foundExistingBook && !shouldUpdate) continue
|
|
|
|
const importBookResponse = await axios.post(`${process.env.BASE_URL}/api/books`, books[i], { headers })
|
|
const importedBook = importBookResponse?.data?.doc || null
|
|
console.log('Added book: ', importedBook)
|
|
}
|
|
}
|
|
|
|
const copyAllExistingDbBooksIntoRepo = async (repoId: number, authToken: string) => {
|
|
const headers = {
|
|
'Authorization': authToken,
|
|
'Content-Type': 'application/json',
|
|
}
|
|
|
|
const allBooksInDbResponse = await axios.get(`${process.env.BASE_URL}/api/books?limit=10000`, { headers })
|
|
const allBooksInDb = (allBooksInDbResponse?.data?.docs || []) as BookRecord[]
|
|
|
|
const allCopiesInDb = await axios.get(`${process.env.BASE_URL}/api/copies?limit=10000`, { headers })
|
|
const alreadyAddedCopies = (allCopiesInDb?.data?.docs || []) as CopyRecord[]
|
|
|
|
for (let i = 0; i < allBooksInDb.length; i++) {
|
|
const foundExistingCopy = alreadyAddedCopies.find(c => {
|
|
return c.book.id === allBooksInDb[i].id
|
|
})
|
|
if (foundExistingCopy) continue
|
|
|
|
const copyProps: Partial<CopyRecord> = {
|
|
book: { id: allBooksInDb[i].id },
|
|
repository: { id: repoId },
|
|
}
|
|
const createCopyResponse = await axios.post(`${process.env.BASE_URL}/api/copies`, copyProps, { headers })
|
|
const importedCopy = createCopyResponse?.data?.doc || null
|
|
console.log('Added copy: ', importedCopy.id)
|
|
}
|
|
}
|
|
|
|
|
|
|
|
const main = async () => {
|
|
console.log("Reading LIbray Thing JSON")
|
|
const libraryThingData = JSON.parse(readFileSync('./libraryThingExport.json', 'utf8')) as LibraryThingExport
|
|
|
|
console.log("Logging in...")
|
|
const authToken = await getAuthToken()
|
|
|
|
|
|
console.log("Parsing Authors...")
|
|
const authors = parseAuthors(libraryThingData)
|
|
console.log("Importing Authors...")
|
|
const authorRecords = await importAuthors(authors, authToken)
|
|
console.log(`Parsed ${authorRecords.length} Authors`)
|
|
|
|
console.log("Parsing Genres...")
|
|
const genres = parseGenre(libraryThingData)
|
|
console.log("Importing Genres...")
|
|
const genreRecords = await importGenres(genres, authToken)
|
|
console.log(`Parsed ${genreRecords.length} Genres`)
|
|
|
|
console.log("Parsing Books...")
|
|
const books = parseBooks(libraryThingData, genreRecords, authorRecords)
|
|
console.log("Importing Books...")
|
|
await importBooks(books, authToken)
|
|
//importBooks(books, authToken, {
|
|
// updateFields: ['asin', 'isbn']
|
|
//})
|
|
console.log("Imported/Updated books")
|
|
|
|
console.log("Copying Books into Repo...")
|
|
await copyAllExistingDbBooksIntoRepo(1, authToken)
|
|
console.log("Copied Books")
|
|
}
|
|
|
|
main()
|