diff --git a/src/bin/cmds/fetchGdeltData.ts b/src/bin/cmds/fetchGdeltData.ts index 4ca7816..4b0c17d 100644 --- a/src/bin/cmds/fetchGdeltData.ts +++ b/src/bin/cmds/fetchGdeltData.ts @@ -62,14 +62,21 @@ async function main( format(current, "yyyyMMddHHmmss"), format(next, "yyyyMMddHHmmss") ); - totalArticles += data.articles.length; - // 檢查 data 的原始類型 - console.log( - "Data Type:", - typeof data, - ", Number of articles:", - data.articles.length - ); + if (data && data.articles) { + totalArticles += data.articles.length; + // 檢查 data 的原始類型 + console.log( + "Data Type:", + typeof data, + ", Number of articles:", + data.articles.length + ); + } else { + console.error("Data or articles is undefined"); + console.log(data); + current = next; + continue; // 跳過當前迴圈迭代 + } // console.log(data); if (data && Array.isArray(data.articles)) { diff --git a/src/utils/fetchContentFromUrls.ts b/src/utils/fetchContentFromUrls.ts index 026ac79..c93e1bf 100644 --- a/src/utils/fetchContentFromUrls.ts +++ b/src/utils/fetchContentFromUrls.ts @@ -69,9 +69,7 @@ async function processBatch(batch: BatchItem[], writeStream: fs.WriteStream, isF } const result = { url: item.url, title: item.title, content }; - if (!isFirst || index > 0) { - writeStream.write(','); - } + writeStream.write(','); writeStream.write(JSON.stringify(result, null, 2)); logger.info(`Extracted content from ${item.url}`); @@ -98,5 +96,18 @@ export async function fetchContentsFromUrls(inputFile: string, outputFile: strin writeStream.write(']'); writeStream.end(); - logger.info(`Data saved to ${outputFile}`); + // 等待寫入完成 + writeStream.on('finish', () => { + // 讀取檔案的第一行 + const fd = fs.openSync(outputFile, 'r+'); + const buffer = Buffer.alloc(1024); + fs.readSync(fd, buffer, 0, buffer.length, 0); + let fileContent = buffer.toString('utf-8'); + // 刪除第一個多餘的逗號 + fileContent = fileContent.replace('[,', '['); + // 寫回檔案的第一行 + fs.writeSync(fd, fileContent, 0, 'utf-8'); + fs.closeSync(fd); + logger.info(`Data saved to ${outputFile}`); + }); }