diff --git a/README.md b/README.md index 84b33c0..68e82b7 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ Go package for automated evaluation of academic papers using LLM-based criteria - Rate limiting with request delay configuration - File-based processing (JSON input/output) - Customizable evaluation criteria +- Robust error handling with failure tracking +- Automatic dump file for failed analyses ## Installation @@ -19,7 +21,7 @@ go get gitea.r8z.us/stwhite/paperprocessor ## API Reference ### ProcessFile -`func ProcessFile(inputPath, outputPath, criteriaPath string, config Config, debug bool) error` +`func ProcessFile(inputPath, outputPath, criteriaPath string, config Config) error` Processes papers from input JSON file and writes results to output JSON file @@ -28,7 +30,6 @@ Parameters: - outputPath: Path to write processing results JSON - criteriaPath: Path to text file with evaluation criteria - config: Configuration settings for API and processing -- debug: Enable debug logging when true Returns: - error: Processing error or nil if successful @@ -54,7 +55,6 @@ err := paperprocessor.ProcessFile( "output/results.json", "criteria.txt", config, - true, // debug mode ) if err != nil { log.Fatal("Processing failed:", err) @@ -106,10 +106,23 @@ Evaluation criteria: "decision": "REJECT", "explanation": "Doesn't meet novelty requirements..." } + ], + "failed": [ + { + "paper": { + "title": "Problematic Paper", + "abstract": "...", + "arxiv_id": "2301.11111" + }, + "error": "invalid decision format", + "output": "" + } ] } ``` +When papers fail processing, they are added to the "failed" list in the output JSON and also written to a `dump.json` file for detailed review. + ## Configuration Options | Parameter | Description | Default | diff --git a/paperprocessor.go b/paperprocessor.go index 90308df..b34e448 100644 --- a/paperprocessor.go +++ b/paperprocessor.go @@ -28,6 +28,11 @@ type PaperResult struct { type ProcessingResult struct { Accepted []PaperResult `json:"accepted"` Rejected []PaperResult `json:"rejected"` + Failed []struct { + Paper Paper `json:"paper"` + Error string `json:"error"` + Output string `json:"output"` + } `json:"failed"` } // Config holds the configuration for the processor @@ -79,7 +84,17 @@ func (p *Processor) ProcessPapers(papers []Paper, criteria string) (*ProcessingR } decision, err := p.evaluatePaper(paper, criteria) if err != nil { - return nil, fmt.Errorf("error processing paper %s: %v", paper.ArxivID, err) + // Instead of returning error, add to failed list + result.Failed = append(result.Failed, struct { + Paper Paper `json:"paper"` + Error string `json:"error"` + Output string `json:"output"` + }{ + Paper: paper, + Error: err.Error(), + Output: "", // We could potentially add the raw LLM output here if needed + }) + continue } paperResult := PaperResult{ @@ -95,6 +110,14 @@ func (p *Processor) ProcessPapers(papers []Paper, criteria string) (*ProcessingR } } + // Write failed analyses to dump file if any exist + if len(result.Failed) > 0 { + dumpData, err := json.MarshalIndent(result.Failed, "", " ") + if err == nil { // Only try to write if marshaling succeeded + ioutil.WriteFile("dump.json", dumpData, 0644) + } + } + return result, nil } @@ -123,8 +146,9 @@ type decisionResult struct { func (p *Processor) evaluatePaper(paper Paper, criteria string) (*decisionResult, error) { prompt := fmt.Sprintf(`Please evaluate the following academic paper against the provided criteria. -Respond with either "ACCEPT" or "REJECT" followed by a brief explanation of your decision. -Do not use markdown emphasis in your response. Keep your response clear and concise. +Respond with either "ACCEPT" or "REJECT" followed by an explanation of your decision. +For ACCEPT decisions, provide a thorough explanation. For REJECT decisions, keep the explanation brief and focused on the key reason. +Do not use markdown, bullet points, or quotes in your response. Keep your response clear and concise. Your response should be in the format: DECISION Explanation @@ -201,10 +225,12 @@ Abstract: %s`, criteria, paper.Title, paper.Abstract) // Handle common prefixes and clean the decision text cleanDecision := rawDecision - for _, prefix := range []string{"DECISION:", "Decision:"} { + for _, prefix := range []string{"DECISION:", "Decision:", "-", "\"", "*"} { cleanDecision = strings.TrimPrefix(cleanDecision, prefix) } cleanDecision = strings.TrimSpace(cleanDecision) + // Remove any remaining quotes + cleanDecision = strings.Trim(cleanDecision, "\"") // Normalize case upperDecision := strings.ToUpper(cleanDecision)