initial commit
This commit is contained in:
ray
@@ -0,0 +1,137 @@
|
||||
// Production rule-based appointment parser from ShopProQuote project.
|
||||
// Extracts multiple appointments from OCR text with no API dependencies.
|
||||
// Handles: phones, VINs (OCR-tolerant), dates, times, durations, vehicles,
|
||||
// advisor+RO stripping, cross-block name carryover, table format detection.
|
||||
// See SKILL.md for the full architecture and pitfall documentation.
|
||||
|
||||
function parseWithRules(ocrText) {
|
||||
var t = ocrText.replace(/\r\n/g, '\n').replace(/\r/g, '\n').trim();
|
||||
if (!t) return [];
|
||||
|
||||
// Strip separator lines and short all-caps headers, KEEP blank lines
|
||||
t = t.split('\n').map(function(line) {
|
||||
var s = line.trim();
|
||||
if (s.length === 0) return ''; // keep blank lines for block splitting
|
||||
if (/^[\-\u2013=_*#]{3,}$/.test(s)) return '';
|
||||
if (s.length < 25 && s === s.toUpperCase() && /^[A-Z\s]+$/.test(s) && s.length > 1) return '';
|
||||
return line;
|
||||
}).join('\n');
|
||||
|
||||
var appointments = [];
|
||||
var phoneRe = /\(?\d{3}\)?[\s.\-]*\d{3}[\s.\-]*\d{4}/;
|
||||
var vinRe = /\b[A-HJ-NPR-Z0-9OIQ]{17}\b/i;
|
||||
function fixVin(v) { return v.toUpperCase().replace(/O/g,'0').replace(/I/g,'1').replace(/Q/g,'0'); }
|
||||
function parseDate(str) { /* see SKILL.md for full implementation */ return null; }
|
||||
var dateRe = /\b(?:\d{4}[\/\-.]\d{1,2}[\/\-.]\d{1,2}|\d{1,2}[\/]\d{1,2}(?:[\/]\d{4})?|(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\s+\d{1,2}(?:[,\s]+\d{4})?)\b/i;
|
||||
var timeRe = /\b(\d{1,2}):(\d{2})\s*(AM|PM|am|pm)?\b|\b(\d{1,2})\s*(AM|PM|am|pm)\b/;
|
||||
var durRe = /\b(\d+)\s*(?:min|minutes?|hrs?|hours?)\b/i;
|
||||
var makesRe = /\b(?:ford|chevy|chevrolet|toyota|honda|nissan|bmw|mercedes|audi|dodge|jeep|gmc|ram|hyundai|kia|subaru|mazda|vw|volkswagen|lexus|acura|infiniti|cadillac|buick|chrysler|lincoln|tesla|volvo)\b/i;
|
||||
var yrRe = /\b(19|20)\d{2}\b/;
|
||||
|
||||
function parseOne(raw, carryName) {
|
||||
var b = raw, appt = {};
|
||||
if (carryName) { appt.customerName = carryName; }
|
||||
// Extract structured fields
|
||||
var pm = b.match(phoneRe); if (pm) { appt.customerPhone = pm[0].replace(/[^\d]/g, ''); b = b.replace(pm[0], ' '); }
|
||||
var vm = b.match(vinRe); if (vm) { appt.vin = fixVin(vm[0]); b = b.replace(vm[0], ' '); }
|
||||
var dm = b.match(dateRe); if (dm) { var pd = parseDate(dm[0]); if (pd) { appt.appointmentDate = pd; b = b.replace(dm[0], ' '); } }
|
||||
var tm = b.match(timeRe);
|
||||
if (tm) { /* parse time to HH:MM, see SKILL.md */ b = b.replace(tm[0], ' '); }
|
||||
var dum = b.match(durRe); if (dum) { var val = parseInt(dum[1]); if (/hr|hour/i.test(dum[0])) val *= 60; appt.duration = val; b = b.replace(dum[0], ' '); }
|
||||
|
||||
// Boundary detection
|
||||
var np = b.match(phoneRe), nv = b.match(vinRe);
|
||||
var myText = b, nextText = '', nextName = null;
|
||||
if (np || nv) {
|
||||
var ni = b.length;
|
||||
if (np) ni = Math.min(ni, b.indexOf(np[0]));
|
||||
if (nv) ni = Math.min(ni, b.indexOf(nv[0]));
|
||||
if (ni > 0 && ni < b.length) {
|
||||
var before = b.substring(0, ni);
|
||||
// Walk back to last multi-space gap before last 1-2 words (tolerate trailing spaces)
|
||||
var gap = before.match(/\s{2,}(?=\S+(?:\s+\S+){0,1}\s*$)/);
|
||||
var splitAt = ni;
|
||||
if (gap) { splitAt = before.lastIndexOf(gap[0]) + gap[0].length; }
|
||||
else {
|
||||
// Fallback: split before last 1-2 capitalized words
|
||||
var nw = before.match(/([A-Z][A-Za-z]+(?:\s+[A-Z][A-Za-z]+){0,1})\s*$/);
|
||||
if (nw && nw[1].length > 2) splitAt = before.length - nw[0].length;
|
||||
}
|
||||
myText = b.substring(0, splitAt);
|
||||
nextText = b.substring(splitAt);
|
||||
}
|
||||
}
|
||||
|
||||
// Split on multi-spaces (preserves column structure)
|
||||
var parts = myText.split(/\s{2,}/).map(function(p) { return p.replace(/\s+/g,' ').trim(); }).filter(function(p) { return p.length > 1; });
|
||||
|
||||
// Strip advisor name + RO code
|
||||
var roCode = null;
|
||||
parts = parts.map(function(p) {
|
||||
var m = p.match(/^([A-Z][a-z]+)\s+(\[[^\]]+\])\s*(.*)/);
|
||||
if (m) { roCode = roCode || m[2]; return m[3] || ''; }
|
||||
return p;
|
||||
}).filter(function(p) { return p.length > 0; });
|
||||
|
||||
// Clean VIN artifacts
|
||||
parts = parts.map(function(p) { return p.replace(/VIN\s*VIN:?\s*/gi,'').replace(/VIN:?\s*VIN\s*/gi,'').trim(); }).filter(function(p) { return p.length > 1; });
|
||||
|
||||
// Assign name (first part, unless carried from previous block)
|
||||
if (!carryName && parts.length > 0) { appt.customerName = parts[0]; parts.shift(); }
|
||||
|
||||
// Vehicle detection: prefer part with BOTH year AND make
|
||||
var vi = -1;
|
||||
for (var i = 0; i < parts.length; i++) {
|
||||
if (yrRe.test(parts[i]) && makesRe.test(parts[i])) { vi = i; break; }
|
||||
}
|
||||
if (vi < 0) for (var i = 0; i < parts.length; i++) {
|
||||
if (yrRe.test(parts[i]) || makesRe.test(parts[i])) { vi = i; break; }
|
||||
}
|
||||
if (vi >= 0) { appt.vehicleInfo = parts[vi]; parts.splice(vi, 1); }
|
||||
if (parts.length > 0) { appt.serviceType = parts.join(' ').replace(/^\s+|\s+$/g, ''); }
|
||||
if (roCode) appt.notes = (appt.notes ? appt.notes + ' ' : '') + 'RO: ' + roCode;
|
||||
|
||||
// Cross-block name carryover: strip trailing all-caps name from service
|
||||
if (appt.serviceType) {
|
||||
var tn = appt.serviceType.match(/\s+([A-Z]{2,}(?:\s+[A-Z]{2,}){1,2})\s*$/);
|
||||
if (tn && tn[1].length > 3) {
|
||||
appt.serviceType = appt.serviceType.substring(0, appt.serviceType.length - tn[0].length).trim();
|
||||
nextName = tn[1];
|
||||
}
|
||||
}
|
||||
|
||||
if (appt.customerName && appt.customerName.length > 1) appointments.push(appt);
|
||||
if (nextText.length > 10) return parseOne(nextText, nextName);
|
||||
return nextName; // return for cross-block carry
|
||||
}
|
||||
|
||||
// Split into blocks, detect tables, parse
|
||||
var blocks = t.split(/\n\s*\n/).filter(function(b) { return b.trim().length > 3; });
|
||||
if (blocks.length === 0) blocks = [t];
|
||||
|
||||
// Table format detection
|
||||
if (blocks.length === 1) {
|
||||
var lines = t.split('\n').filter(function(l) { return l.trim().length > 0; });
|
||||
if (lines.length >= 3 && lines.length <= 40) {
|
||||
var fl = lines[0].trim();
|
||||
var hw = (fl.match(/\b(name|phone|date|time|service|vehicle|vin|customer|appointment|mileage|advisor|status)\b/gi) || []).length;
|
||||
var hd = phoneRe.test(fl) || vinRe.test(fl);
|
||||
// Only strip if 3+ header words AND no phone/VIN (data guard)
|
||||
if (hw >= 3 && !hd && lines.length > 2) lines = lines.slice(1);
|
||||
var wc = lines.map(function(l) { return l.trim().split(/\s+/).length; });
|
||||
var avg = wc.reduce(function(a,b) { return a+b; }, 0) / wc.length;
|
||||
if (wc.every(function(w) { return Math.abs(w - avg) <= 3; }) && lines.length >= 2) {
|
||||
lines.forEach(function(line) { parseOne(line.replace(/\t/g, ' ').replace(/\n/g, ' ')); });
|
||||
return appointments;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Standard block-by-block with cross-block name carry
|
||||
var carry = null;
|
||||
blocks.forEach(function(block) {
|
||||
carry = parseOne(block.replace(/\t/g, ' ').replace(/\n/g, ' '), carry);
|
||||
});
|
||||
|
||||
return appointments;
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
# OCR Parser Pattern Catalog
|
||||
|
||||
Extraction regexes and test cases for the `ocr-data-extraction` skill.
|
||||
|
||||
## Phone Numbers
|
||||
|
||||
```regex
|
||||
/\(?\d{3}\)?[\s.\-]*\d{3}[\s.\-]*\d{4}/
|
||||
```
|
||||
|
||||
Matches: `(555) 123-4567`, `555-123-4567`, `555.123.4567`, `8653353874`, `(865) 235-0156`
|
||||
|
||||
Normalize: strip all non-digits after match.
|
||||
|
||||
## VIN (17-char)
|
||||
|
||||
```regex
|
||||
/\b[A-HJ-NPR-Z0-9OIQ]{17}\b/i
|
||||
```
|
||||
|
||||
Lenient: accepts O (common OCR error for 0), I (for 1), Q (for 0).
|
||||
Normalize: `v.toUpperCase().replace(/O/g,'0').replace(/I/g,'1').replace(/Q/g,'0')`
|
||||
|
||||
Standard VIN: 17 chars, alphanumeric, no I/O/Q in real VINs, but OCR produces them.
|
||||
|
||||
## Dates
|
||||
|
||||
Priority order of patterns:
|
||||
```js
|
||||
// 1. YYYY-MM-DD or YYYY/MM/DD
|
||||
/(\d{4})[\/\-.](\d{1,2})[\/\-.](\d{1,2})/
|
||||
|
||||
// 2. MM/DD/YYYY or MM/DD (year defaults to current)
|
||||
/(\d{1,2})[\/](\d{1,2})(?:[\/](\d{4}))?/
|
||||
|
||||
// 3. "Jan 15, 2024" or "January 15 2024"
|
||||
/(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\s+(\d{1,2})(?:[,\s]+(\d{4}))?/i
|
||||
```
|
||||
|
||||
Normalize all to `YYYY-MM-DD`.
|
||||
|
||||
## Times
|
||||
|
||||
```regex
|
||||
/\b(\d{1,2}):(\d{2})\s*(AM|PM|am|pm)?\b|\b(\d{1,2})\s*(AM|PM|am|pm)\b/
|
||||
```
|
||||
|
||||
Matches: `9:00 AM`, `14:00`, `1:30PM`, `8am`, `8 AM`
|
||||
Normalize to 24hr `HH:MM`. PM adds 12 unless hour=12. AM at 12 becomes 00.
|
||||
|
||||
## Duration
|
||||
|
||||
```regex
|
||||
/\b(\d+)\s*(?:min|minutes?|hrs?|hours?)\b/i
|
||||
```
|
||||
|
||||
Matches: `60 min`, `1 hour`, `2 hours`, `90min`
|
||||
If "hour/hr" in match text, multiply value by 60.
|
||||
|
||||
## Vehicle Detection
|
||||
|
||||
Year pattern: `/\b(19|20)\d{2}\b/`
|
||||
|
||||
Make keywords (case-insensitive):
|
||||
```
|
||||
ford|chevy|chevrolet|toyota|honda|nissan|bmw|mercedes|audi|dodge|jeep|gmc|ram|hyundai|kia|subaru|mazda|vw|volkswagen|lexus|acura|infiniti|cadillac|buick|chrysler|lincoln|tesla|volvo
|
||||
```
|
||||
|
||||
## Advisor + RO Code
|
||||
|
||||
```regex
|
||||
/^([A-Z][a-z]+)\s+(\[[^\]]+\])\s+(.*)/
|
||||
```
|
||||
|
||||
Strip group 1 (advisor name) and group 2 (RO code). Save group 2 as note. Return group 3 (service description).
|
||||
|
||||
## Boundary Detection
|
||||
|
||||
### Primary: multi-space gap before name+phone
|
||||
```regex
|
||||
/\s{2,}(?=\S+(?:\s+\S+){0,1}\s*$)/
|
||||
```
|
||||
Finds 2+ spaces before the last 1-2 words that precede a phone/VIN marker.
|
||||
|
||||
### Fallback: capitalized name before phone
|
||||
```regex
|
||||
/([A-Z][A-Za-z]+(?:\s+[A-Z][A-Za-z]+){0,1})\s*$/
|
||||
```
|
||||
When no multi-space gap exists, detect last 1-2 capitalized words as the next appointment's name.
|
||||
|
||||
## Header/Noise Filters
|
||||
|
||||
```regex
|
||||
# Separator lines
|
||||
/^[\-–=_*#]{3,}$/
|
||||
|
||||
# Short all-caps headers (< 25 chars)
|
||||
/^[A-Z\s]+$/ (with length check)
|
||||
|
||||
# Known header words (case-insensitive)
|
||||
/^(schedule|appointments|roster|calendar|upcoming|date|time|customer|name|phone)$/i
|
||||
|
||||
# Table header row detection
|
||||
/\b(name|phone|date|time|service|vehicle|vin|customer|appointment|mileage|advisor|status)\b/i
|
||||
```
|
||||
|
||||
## Verified Test Cases
|
||||
|
||||
### 1. Two appointments with column spacing
|
||||
```
|
||||
John Smith (555) 123-4567 2024-06-15 9:00 AM Oil Change 60 min 2018 Toyota Camry
|
||||
|
||||
Jane Doe (555) 987-6543 06/20/2024 2:00 PM Brake Inspection 90 min 2020 Honda Civic
|
||||
```
|
||||
→ 2 appointments, all fields correct
|
||||
|
||||
### 2. Headers with separators
|
||||
```
|
||||
SCHEDULE
|
||||
--------
|
||||
Robert Brown 865-555-1234 Jan 15, 2024 8:00am Tire Rotation 1 hour Toyota Tacoma
|
||||
|
||||
Sarah Wilson (423)555-6789 03/01/2024 1:30PM AC Repair 2 hours Jeep Grand Cherokee
|
||||
```
|
||||
→ 2 appointments, headers/separators stripped
|
||||
|
||||
### 3. Single appointment
|
||||
```
|
||||
Mike Johnson 2024-07-01 10:30 AM Transmission Fluid 2022 Ford F-150
|
||||
```
|
||||
→ 1 appointment, vehicle correctly separated from service
|
||||
|
||||
### 4. Table with header row
|
||||
```
|
||||
Name Phone Date Time Service Vehicle
|
||||
John Smith 555-123-4567 2024-06-15 9:00 AM Oil Change 2018 Toyota Camry
|
||||
Jane Doe 555-987-6543 06/20/2024 2:00 PM Brake Inspection 2020 Honda Civic
|
||||
```
|
||||
→ 2 appointments, header row skipped
|
||||
|
||||
### 5. Real shop screenshot (3 appointments, OCR-garbled)
|
||||
```
|
||||
Gary Bowers 8653353874 Rrahman [REP] Tailgate wiring harness recall 2023 Honda RIDGELINE VIN SEPYK3F70PBO11523 60min garyb9623@gmail.com
|
||||
|
||||
HITESHKUMAR PATEL (865)235-0156 Rrahman [23-046:6MX00] Odyssey MOST 2023 Honda ODYSSEY VIN SENRLEHB9PB042710 60min
|
||||
|
||||
STEPHEN LOWERY (865)384-6984 Rrahman [MA10] Oil w/Filter Change 2013 HONDA CROSSTOUR VIN SJ6TF1HS4DLO01541 60min
|
||||
```
|
||||
→ 3 appointments, VINs corrected (O→0), RO codes extracted, advisor names stripped
|
||||
@@ -0,0 +1,112 @@
|
||||
# OCR Regex Catalog — Appointment Parsing
|
||||
|
||||
Full extraction patterns from the ShopProQuote production parser, with OCR corruption variants and test cases.
|
||||
|
||||
## Structured Field Patterns
|
||||
|
||||
| Field | Regex | Notes |
|
||||
|-------|-------|-------|
|
||||
| Phone | `\(?\d{3}\)?[\s.\-]*\d{3}[\s.\-]*\d{4}` | Forgiving of (xxx), xxx-xxx, xxx.xxx |
|
||||
| VIN | `\b[A-HJ-NPR-Z0-9OIQ]{17}\b/i` | Accepts O→0, I→1, Q→0 OCR errors |
|
||||
| Email | `\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b` | Extract BEFORE VIN to prevent false matches |
|
||||
| Date | `\b(?:\d{4}[\/\-.]\d{1,2}[\/\-.]\d{1,2}|\d{1,2}[\/]\d{1,2}(?:[\/]\d{4})?|(?:Jan|Feb|...)[a-z]*\s+\d{1,2}(?:[,\s]+\d{4})?)\b` | 3 formats: ISO, US slash, month name |
|
||||
| Time | `\b(\d{1,2}):(\d{2})\s*(AM|PM)?\b|\b(\d{1,2})\s*(AM|PM)\b` | 9:00 AM, 1:30PM, 14:00, 8am |
|
||||
| Duration | `\b(\d+(?:\.\d+)?)\s*(?:min|minutes?|hrs?|hours?)\b` | 0.5 hrs → 30 min, 60min → 60 |
|
||||
| OP Code | `\[([A-Z0-9\-:]+)\]` | [REP], [MA10], [23-046:6MX00] |
|
||||
| Vehicle | `(\d{4})\s+([A-Za-z]+)\s+([A-Za-z0-9\-]+)` | 2023 Honda RIDGELINE |
|
||||
| Vehicle fallback | `(\d{4})\s+([A-Za-z]+)` | 2023 Honda |
|
||||
| Row anchor | `(\d{1,2}):(\d{2})\s*(AM|PM)\s+(\d+(?:\.\d+)?)\s*hrs?` | 9:00AM 0.5 hrs |
|
||||
|
||||
## Known Makes
|
||||
|
||||
```
|
||||
ford|chevy|chevrolet|toyota|honda|nissan|bmw|mercedes|audi|dodge|jeep|gmc|ram|hyundai|kia|subaru|mazda|vw|volkswagen|lexus|acura|infiniti|cadillac|buick|chrysler|lincoln|tesla|volvo
|
||||
```
|
||||
|
||||
## Advisor/Noise Stripping Patterns
|
||||
|
||||
### Known Advisor Names
|
||||
```
|
||||
Rrahman Grajqevci, Rrshman, Grajqevci, Grokevel
|
||||
```
|
||||
|
||||
### OCR Corruption Variants
|
||||
```
|
||||
eE 3 zoom, Eee om 3 gem, eE\s*3\s*zoom, Eee\s*om\s*3\s*gem
|
||||
```
|
||||
|
||||
### Junk Tokens
|
||||
```
|
||||
RO, VIN, os, b=
|
||||
```
|
||||
|
||||
### Header Metadata (strip before parsing)
|
||||
```
|
||||
/^(Monday|Tuesday|...|Sunday)\s+[A-Z][a-z]{2}\s+\d{1,2}/i // "Wednesday Jun 18, 2025"
|
||||
/^(Advisor|Refresh|Weekly|Daily)(\s|$)/i // "Weekly", "Advisor"
|
||||
```
|
||||
|
||||
## Service Extraction Rule
|
||||
|
||||
```js
|
||||
// Stop at newline — critical to prevent vehicle info absorption
|
||||
var svcRe = new RegExp('\\[' + opCode + '\\]\\s*([^\\n\\[]*)', 'i');
|
||||
```
|
||||
|
||||
**Wrong** (captures vehicle on next line):
|
||||
```js
|
||||
var svcRe = new RegExp('\\[' + opCode + '\\]\\s*([^\\[]*)', 'i');
|
||||
```
|
||||
|
||||
## Block Splitting Hierarchy
|
||||
|
||||
1. **Row-anchor split** (preferred): Find Time+Duration patterns → split at each boundary
|
||||
2. **Blank-line fallback**: `t.split(/\n\s*\n/)` when < 2 anchors found
|
||||
3. **Table fallback**: Uniform word counts across lines → treat each line as a record
|
||||
|
||||
## VIN OCR Fix
|
||||
|
||||
```js
|
||||
function fixVin(v) {
|
||||
return v.toUpperCase().replace(/O/g,'0').replace(/I/g,'1').replace(/Q/g,'0');
|
||||
}
|
||||
```
|
||||
|
||||
## Cross-Block Name Carry
|
||||
|
||||
```js
|
||||
// Return value from parseOne() is nextName — passes between blocks
|
||||
if (appt.serviceType) {
|
||||
var tn = appt.serviceType.match(/\s+([A-Z]{2,}(?:\s+[A-Z]{2,}){1,2})\s*$/);
|
||||
if (tn && tn[1].length > 3) {
|
||||
appt.serviceType = appt.serviceType.substring(0, appt.serviceType.length - tn[0].length).trim();
|
||||
nextName = tn[1];
|
||||
}
|
||||
}
|
||||
return nextName; // fed as carryName to next block
|
||||
```
|
||||
|
||||
## Test Cases
|
||||
|
||||
### T5: 3-appointment screenshot with blank lines
|
||||
```
|
||||
9:00AM 0.5 hrs RO Gary Bowers (865) 335-3874 Rrahman Grajqevci [REP] Tailgate wiring harness connector replacement garyb9623@gmail.com
|
||||
2023 Honda RIDGELINE VIN SEPYK3F70PB011523
|
||||
|
||||
10:30AM 1.0 hrs RO HITESHKUMAR PATEL 865-235-0156 Rrahman Grajqevci [23-046:6MX00] Oil change and tire rotation
|
||||
2013 Honda ODYSSEY VIN SENRLEHB9PB042710
|
||||
|
||||
11:45AM 0.5 hrs RO STEPHEN LOWERY 865-384-6984 Rrahman Grajqevci [MA10] Replace front brake pads
|
||||
2013 Honda CROSSTOUR VIN SJ6TF1HS4DL001541
|
||||
```
|
||||
|
||||
### T6: Row-anchor (no blank lines, Monday header)
|
||||
```
|
||||
Wednesday Jun 18, 2025 Weekly Advisor Refresh
|
||||
9:00AM 0.5 hrs RO Gary Bowers (865) 335-3874 Rrahman Grajqevci [REP] Tailgate wiring
|
||||
2023 Honda RIDGELINE VIN 5FPYK3F70PB011523 garyb9623@gmail.com
|
||||
10:30AM 1.0 hrs RO HITESHKUMAR PATEL 865-235-0156 Rrahman Grajqevci [23-046:6MX00] Oil change
|
||||
2013 Honda ODYSSEY VIN SENRLEHB9PB042710
|
||||
11:45AM 0.5 hrs RO STEPHEN LOWERY 865-384-6984 Rrahman Grajqevci [MA10] Brake pads
|
||||
2013 Honda CROSSTOUR VIN SJ6TF1HS4DL001541
|
||||
```
|
||||
Reference in New Issue
Block a user