Modded system to use dates and search for filenames by default.

This commit is contained in:
Steve White 2025-01-24 09:56:44 -06:00
parent 9396e2da3a
commit a0da416f88
3 changed files with 28 additions and 11 deletions

3
.gitignore vendored
View File

@ -9,6 +9,7 @@
/arxiv-processor/arxiv-processor
/json2md/json2md
/llm_processor/llm_processor
paper-system
# Dependency directories
vendor/
@ -27,6 +28,8 @@ temp_*.json
.env
# Generated output files
*.md
*.json
papers.json
newpapers.md.gz
arxiv-processor/newpapers.json

36
main.go
View File

@ -169,10 +169,11 @@ func main() {
if err != nil {
log.Fatalf("Failed to marshal papers: %v", err)
}
if err := os.WriteFile("papers.json", papersJSON, 0644); err != nil {
jsonName := fmt.Sprintf("%s-%s-%s-papers.json", *start, *end, *search)
if err := os.WriteFile(jsonName, papersJSON, 0644); err != nil {
log.Fatalf("Failed to save papers JSON: %v", err)
}
log.Printf("Successfully saved papers to papers.json")
log.Printf("Successfully saved papers to %s", jsonName)
}
// Print paper titles for verification
@ -182,12 +183,17 @@ func main() {
}
// Save papers to temporary file for LLM processing
tempInput := "temp_input.json"
tempInput, err := os.CreateTemp("", "arxiv-process-*.tmp")
if err != nil {
log.Fatalf("Failed to create temp file: %v", err)
}
defer os.Remove(tempInput.Name())
tempJSON, err := json.Marshal(llmPapers)
if err != nil {
log.Fatalf("Failed to marshal papers for LLM: %v", err)
}
if err := os.WriteFile(tempInput, tempJSON, 0644); err != nil {
if err := os.WriteFile(tempInput.Name(), tempJSON, 0644); err != nil {
log.Fatalf("Failed to save temp input JSON: %v", err)
}
@ -201,30 +207,38 @@ func main() {
llmProcessor := processor.NewProcessor(*model, 32, apiKey) // 32 = batch size from README
log.Printf("Initialized LLM processor with model %s", *model)
tempOutput := "temp_output.json"
tempOutput, err := os.CreateTemp("", "arxiv-process-*.tmp")
if err != nil {
log.Fatalf("Failed to create temp file: %v", err)
}
defer os.Remove(tempOutput.Name())
log.Printf("Processing papers with criteria from %s", *criteriaFile)
if err := llmProcessor.ProcessPapers(ctx, tempInput, tempOutput, *criteriaFile, 1*time.Second); err != nil {
if err := llmProcessor.ProcessPapers(ctx, tempInput.Name(), tempOutput.Name(), *criteriaFile, 1*time.Second); err != nil {
log.Fatalf("LLM processing failed: %v", err)
}
log.Printf("LLM processing complete, results saved to %s", tempOutput)
// Generate markdown
log.Printf("Generating markdown output")
decisions, err := lib.ProcessJSONFile(tempOutput)
decisions, err := lib.ProcessJSONFile(tempOutput.Name())
if err != nil {
log.Fatalf("Failed to process JSON: %v", err)
}
log.Printf("Processed decisions: %d accepted, %d rejected", len(decisions.Accepted), len(decisions.Rejected))
defaultOutput := fmt.Sprintf("%s-%s-%s-papers.md", *start, *end, *search)
if *output == "papers.md" && usingArxiv {
*output = defaultOutput
}
if err := lib.GenerateMarkdown(decisions, *output); err != nil {
log.Fatalf("Markdown generation failed: %v", err)
}
log.Printf("Generated markdown output at %s", *output)
// Cleanup temp files
os.Remove(tempInput)
os.Remove(tempOutput)
log.Printf("Cleaned up temporary files")
// Temp files cleaned up by defer statements
log.Printf("Cleanup complete")
log.Printf("Process complete. Results saved to %s", *output)
}

Binary file not shown.