libraeryThing-to-midrashim-import/index.ts

import { readFileSync, writeFileSync } from 'fs';
import axios from 'axios';

type LibraryThingAuthor = {
  lf: string,
  fl: string,
  role: string,
}

type LibraryThingBook = {
  books_id: string,
  title: string,
  authors: LibraryThingAuthor[],
  pages: string,
  lcc: {
    code: string
  },
  publication: string,
  publicationDate: string,
  genre: string[],
  summary: string,
  language: string[],
}

type LibraryThingExport = Record<string, LibraryThingBook>

type AuthorMap = Map<string, LibraryThingAuthor>

type AuthorRecord = LibraryThingAuthor & {
  id: number,
  updatedAt: string,
  createdAt: string,
}

type GenreImport = {
  name: string,
}

type GenreRecord = {
  id: number,
  name: string,
  updatedAt: string,
  createdAt: string,
}

type BookImport = {
  books_id: string,
  title: string,
  pages?: number,
  lcc?: string,
  publication?: string,
  genre?: { id: number }[], // id of the genre already stored in DB
  summary?: string,
  authors?: { id: number }[], // id of the author already stored in DB
}

type BookRecord = BookImport & {
  id: number
  genre: GenreRecord[]
}


const getAuthToken = async () => {
  const creds = {
    email: process.env.EMAIL,
    password: process.env.PASSWORD,
  }

  const loginRequest = await axios.post(
    `${process.env.BASE_URL}/api/users/login`,
    creds,
  )

  const authToken = loginRequest.data.token
  return authToken
}


const makeAuthorKey = (author: LibraryThingAuthor) => {
  return `${author.role}: ${author.lf}`
}

const parseAuthors = (libraryThingData: LibraryThingExport): AuthorMap => {
  const authors = new Map<string, LibraryThingAuthor>() as AuthorMap

  const bookIds = Object.keys(libraryThingData)
  bookIds.forEach(bookId => {
    const currentBook = libraryThingData[bookId]
    currentBook.authors.forEach(a => {
      const authorKey = makeAuthorKey(a)
      const existingAuthor = authors.get(authorKey)
      if (!existingAuthor) authors.set(authorKey, a)
    })
  });

  return authors
}

const importAuthors = async (authors: AuthorMap, authToken: string) => {
  const headers = {
    'Authorization': authToken,
    'Content-Type': 'application/json',
  }

  const allAuthorsInDb = await axios.get(`${process.env.BASE_URL}/api/authors?limit=10000`, { headers })

  const alreadyAddedAuthors = (allAuthorsInDb?.data?.docs || []) as AuthorRecord[]

  const importPromises = [...authors.values()].map(async (a) => {
    const foundAuthor = alreadyAddedAuthors.find(addedAuthor => {
      if (addedAuthor.lf !== a.lf) return false

      if (!a.role && !addedAuthor.role) return true

      if (a.role && a.role === addedAuthor.role) return true

      return false
    })

    if (foundAuthor) {
      //console.log('Author already exists - ', makeAuthorKey(foundAuthor))
      return
    }


    return await axios.post(
      `${process.env.BASE_URL}/api/authors`,
      a, { headers, withCredentials: true }
    ).then(saveResponse => {
      const savedAuthor = saveResponse?.data?.doc as AuthorRecord || null
      if (savedAuthor?.id) {
        console.log('Added Author:', savedAuthor.lf)
        alreadyAddedAuthors.push(savedAuthor)
      }
    })
  });

  await Promise.allSettled(importPromises)
  return alreadyAddedAuthors
}

const parseGenre = (libraryThingData: LibraryThingExport) => {
  const parsedGenres: string[] = []

  const bookIds = Object.keys(libraryThingData)
  bookIds.forEach(bookId => {
    const currentBook = libraryThingData[bookId]
    currentBook.genre?.forEach(g => {
      if (!parsedGenres.includes(g)) parsedGenres.push(g)
    })
  });

  return parsedGenres
}

const importGenres = async (genres: string[], authToken: string) => {
  const headers = {
    'Authorization': authToken,
    'Content-Type': 'application/json',
  }

  const allGenresInDb = await axios.get(`${process.env.BASE_URL}/api/genre?limit=10000`, { headers })

  const alreadyAddedGenres = (allGenresInDb?.data?.docs || []) as GenreRecord[]

  const importGenrePromises = genres.map(async g => {
    const foundGenre = alreadyAddedGenres.find(added => added.name === g)
    if (foundGenre) new Promise(() => { })

    return axios.post(
      `${process.env.BASE_URL}/api/genre`,
      { name: g },
      { headers },
    ).then(res => {
      const importedGenre = res?.data?.doc || null
      if (importedGenre) alreadyAddedGenres.push(importedGenre)
    })
  });

  await Promise.allSettled(importGenrePromises)
  return alreadyAddedGenres
}

const parseBooks = (libraryThingData: LibraryThingExport, genreRecords: GenreRecord[], authorRecords: AuthorRecord[]) => {
  const books: BookImport[] = []

  const bookIds = Object.keys(libraryThingData)
  bookIds.forEach(bookId => {
    const curr = libraryThingData[bookId]

    const parsedBook: BookImport = {
      books_id: curr.books_id,
      title: curr.title,
      lcc: curr.lcc?.code || '',
      publication: curr.publication,
      summary: curr.summary,
      genre: [] as { id: number }[],
      authors: [] as { id: number }[]
    }

    const formatedPages = parseInt(curr.pages?.trim() || '', 10)
    if (formatedPages) parsedBook.pages = formatedPages

    curr.genre?.forEach(g => {
      const matchingGenreRecord = genreRecords.find(rec => rec.name === g)
      if (matchingGenreRecord) parsedBook.genre?.push({ id: matchingGenreRecord.id })
    })

    curr.authors.forEach(a => {
      const matchingAuthor = authorRecords.find(match => {
        if (match.lf !== a.lf) return false
        if (!a.role && !match.role) return true
        if (a.role && a.role === match.role) return true
        return false
      })

      if (matchingAuthor) parsedBook.authors?.push({ id: matchingAuthor.id })
    })

    books.push(parsedBook)
  });

  return books
}

const importBooks = async (books: BookImport[], authToken: string) => {
  const headers = {
    'Authorization': authToken,
    'Content-Type': 'application/json',
  }

  const allBooksInDb = await axios.get(`${process.env.BASE_URL}/api/books?limit=10000`, { headers })
  const alreadyAddedBooks = (allBooksInDb?.data?.docs || []) as BookRecord[]
  books.forEach(async b => {
    const foundExistingBook = alreadyAddedBooks.find(added => added.books_id === b.books_id)
    if (foundExistingBook) return

    const importBookResponse = await axios.post(`${process.env.BASE_URL}/api/books`, b, { headers })
    const importedBook = importBookResponse?.data?.doc || null
    console.log('Added book: ', importedBook)
  });

}


const main = async () => {
  const libraryThingData = JSON.parse(readFileSync('./libraryThingExport.json', 'utf8')) as LibraryThingExport
  const authToken = await getAuthToken()

  const authors = parseAuthors(libraryThingData)
  const authorRecords = await importAuthors(authors, authToken)

  const genres = parseGenre(libraryThingData)
  const genreRecords = await importGenres(genres, authToken)

  const books = parseBooks(libraryThingData, genreRecords, authorRecords)
  importBooks(books, authToken)
}

main()