Armored response parsing, dump failed analysis to a file

README.md

@@ -9,6 +9,8 @@ Go package for automated evaluation of academic papers using LLM-based criteria
 - Rate limiting with request delay configuration
 - File-based processing (JSON input/output)
 - Customizable evaluation criteria
+- Robust error handling with failure tracking
+- Automatic dump file for failed analyses
 
 ## Installation
 
@@ -19,7 +21,7 @@ go get gitea.r8z.us/stwhite/paperprocessor
 ## API Reference
 
 ### ProcessFile
-`func ProcessFile(inputPath, outputPath, criteriaPath string, config Config, debug bool) error`
+`func ProcessFile(inputPath, outputPath, criteriaPath string, config Config) error`
 
 Processes papers from input JSON file and writes results to output JSON file
 
@@ -28,7 +30,6 @@ Parameters:
 - outputPath: Path to write processing results JSON
 - criteriaPath: Path to text file with evaluation criteria
 - config: Configuration settings for API and processing
-- debug: Enable debug logging when true
 
 Returns:
 - error: Processing error or nil if successful
@@ -54,7 +55,6 @@ err := paperprocessor.ProcessFile(
     "output/results.json",
     "criteria.txt",
     config,
-    true, // debug mode
 )
 if err != nil {
     log.Fatal("Processing failed:", err)
@@ -106,10 +106,23 @@ Evaluation criteria:
             "decision": "REJECT",
             "explanation": "Doesn't meet novelty requirements..."
         }
+    ],
+    "failed": [
+        {
+            "paper": {
+                "title": "Problematic Paper",
+                "abstract": "...",
+                "arxiv_id": "2301.11111"
+            },
+            "error": "invalid decision format",
+            "output": ""
+        }
     ]
 }
 ```
 
+When papers fail processing, they are added to the "failed" list in the output JSON and also written to a `dump.json` file for detailed review.
+
 ## Configuration Options
 
 | Parameter       | Description                          | Default       |

paperprocessor.go

@@ -28,6 +28,11 @@ type PaperResult struct {
 type ProcessingResult struct {
 	Accepted []PaperResult `json:"accepted"`
 	Rejected []PaperResult `json:"rejected"`
+	Failed   []struct {
+		Paper  Paper  `json:"paper"`
+		Error  string `json:"error"`
+		Output string `json:"output"`
+	} `json:"failed"`
 }
 
 // Config holds the configuration for the processor
@@ -79,7 +84,17 @@ func (p *Processor) ProcessPapers(papers []Paper, criteria string) (*ProcessingR
 		}
 		decision, err := p.evaluatePaper(paper, criteria)
 		if err != nil {
-			return nil, fmt.Errorf("error processing paper %s: %v", paper.ArxivID, err)
+			// Instead of returning error, add to failed list
+			result.Failed = append(result.Failed, struct {
+				Paper  Paper  `json:"paper"`
+				Error  string `json:"error"`
+				Output string `json:"output"`
+			}{
+				Paper:  paper,
+				Error:  err.Error(),
+				Output: "", // We could potentially add the raw LLM output here if needed
+			})
+			continue
 		}
 
 		paperResult := PaperResult{
@@ -95,6 +110,14 @@ func (p *Processor) ProcessPapers(papers []Paper, criteria string) (*ProcessingR
 		}
 	}
 
+	// Write failed analyses to dump file if any exist
+	if len(result.Failed) > 0 {
+		dumpData, err := json.MarshalIndent(result.Failed, "", "  ")
+		if err == nil { // Only try to write if marshaling succeeded
+			ioutil.WriteFile("dump.json", dumpData, 0644)
+		}
+	}
+
 	return result, nil
 }
 
@@ -123,8 +146,9 @@ type decisionResult struct {
 
 func (p *Processor) evaluatePaper(paper Paper, criteria string) (*decisionResult, error) {
 	prompt := fmt.Sprintf(`Please evaluate the following academic paper against the provided criteria.
-Respond with either "ACCEPT" or "REJECT" followed by a brief explanation of your decision. 
+Respond with either "ACCEPT" or "REJECT" followed by an explanation of your decision. 
-Do not use markdown emphasis in your response. Keep your response clear and concise.
+For ACCEPT decisions, provide a thorough explanation. For REJECT decisions, keep the explanation brief and focused on the key reason.
+Do not use markdown, bullet points, or quotes in your response. Keep your response clear and concise.
 Your response should be in the format:
 DECISION
 Explanation
@@ -201,10 +225,12 @@ Abstract: %s`, criteria, paper.Title, paper.Abstract)
 
 	// Handle common prefixes and clean the decision text
 	cleanDecision := rawDecision
-	for _, prefix := range []string{"DECISION:", "Decision:"} {
+	for _, prefix := range []string{"DECISION:", "Decision:", "-", "\"", "*"} {
 		cleanDecision = strings.TrimPrefix(cleanDecision, prefix)
 	}
 	cleanDecision = strings.TrimSpace(cleanDecision)
+	// Remove any remaining quotes
+	cleanDecision = strings.Trim(cleanDecision, "\"")
 
 	// Normalize case
 	upperDecision := strings.ToUpper(cleanDecision)