|
10 | 10 | - Converts to NIST-style JSON format with proper metadata |
11 | 11 | - Handles control text, discussions, and related controls |
12 | 12 | - Provides comprehensive summary statistics |
| 13 | +- Filters out controls with no or empty text attributes |
13 | 14 |
|
14 | 15 | Usage: |
15 | 16 | python sl5_control_catalog_extractor.py <pdf_file> |
@@ -77,68 +78,56 @@ def extract_controls(self) -> List[Dict]: |
77 | 78 | self._process_page(page, page_num + 1) |
78 | 79 |
|
79 | 80 | doc.close() |
80 | | - |
81 | | - # Sort controls by ID |
82 | | - self.controls.sort(key=lambda x: self._sort_key(x['id'])) |
83 | | - |
| 81 | + print(f"Extracted {len(self.controls)} controls") |
84 | 82 | return self.controls |
85 | 83 |
|
86 | 84 | except Exception as e: |
87 | 85 | print(f"Error processing PDF: {e}") |
88 | 86 | return [] |
89 | 87 |
|
90 | | - def _sort_key(self, control_id: str) -> tuple: |
91 | | - """Generate sort key for control ID.""" |
92 | | - # Parse control ID like "SC-61" or "SC-61(1)" |
93 | | - match = re.match(r'^([A-Z]{2})-(\d+)(?:\((\d+)\))?', control_id) |
94 | | - if match: |
95 | | - family, base_num, enhancement = match.groups() |
96 | | - return (family, int(base_num), int(enhancement) if enhancement else 0) |
97 | | - return (control_id, 0, 0) |
98 | | - |
99 | 88 | def _process_page(self, page, page_num: int): |
100 | | - """Process a single page.""" |
| 89 | + """Process a single page extracting control information.""" |
101 | 90 | try: |
102 | | - text_dict = page.get_text("dict") |
103 | | - formatted_text = self._extract_formatted_text(text_dict) |
104 | | - self._find_controls_and_attributes(formatted_text, page_num) |
| 91 | + # Get formatted text with style information |
| 92 | + formatted_text = self._get_formatted_text(page) |
| 93 | + self._extract_controls_from_text(formatted_text, page_num) |
105 | 94 | except Exception as e: |
106 | 95 | if self.debug_mode: |
107 | 96 | print(f"Error processing page {page_num}: {e}") |
108 | 97 |
|
109 | | - def _extract_formatted_text(self, text_dict: dict) -> List[dict]: |
110 | | - """Extract text with formatting information.""" |
111 | | - formatted_text = [] |
| 98 | + def _get_formatted_text(self, page) -> List[Dict]: |
| 99 | + """Extract text with formatting information from the page.""" |
| 100 | + text_dict = page.get_text("dict") |
| 101 | + formatted_lines = [] |
112 | 102 |
|
113 | | - for block in text_dict.get("blocks", []): |
114 | | - if block.get("type") == 0: # Text block |
115 | | - for line in block.get("lines", []): |
| 103 | + for block in text_dict["blocks"]: |
| 104 | + if "lines" in block: |
| 105 | + for line in block["lines"]: |
116 | 106 | line_text = "" |
117 | | - line_formats = [] |
| 107 | + formats = [] |
118 | 108 |
|
119 | | - for span in line.get("spans", []): |
120 | | - text = span.get("text", "") |
121 | | - flags = span.get("flags", 0) |
| 109 | + for span in line["spans"]: |
| 110 | + span_text = span["text"] |
| 111 | + line_text += span_text |
122 | 112 |
|
123 | | - line_text += text |
124 | | - line_formats.append({ |
125 | | - "text": text, |
126 | | - "bold": bool(flags & 16), |
127 | | - "underline": bool(flags & 2), |
128 | | - "font": span.get("font", ""), |
129 | | - "flags": flags |
| 113 | + # Track format information |
| 114 | + formats.append({ |
| 115 | + "text": span_text, |
| 116 | + "bold": "Bold" in span["font"] or span["flags"] & 2**4, |
| 117 | + "italic": "Italic" in span["font"] or span["flags"] & 2**1, |
| 118 | + "size": span["size"] |
130 | 119 | }) |
131 | 120 |
|
132 | 121 | if line_text.strip(): |
133 | | - formatted_text.append({ |
| 122 | + formatted_lines.append({ |
134 | 123 | "text": line_text.strip(), |
135 | | - "formats": line_formats |
| 124 | + "formats": formats |
136 | 125 | }) |
137 | 126 |
|
138 | | - return formatted_text |
| 127 | + return formatted_lines |
139 | 128 |
|
140 | | - def _find_controls_and_attributes(self, formatted_text: List[dict], page_num: int): |
141 | | - """Find controls and attributes in the formatted text.""" |
| 129 | + def _extract_controls_from_text(self, formatted_text: List[Dict], page_num: int): |
| 130 | + """Extract control information from formatted text.""" |
142 | 131 | processing_enhancements = False |
143 | 132 |
|
144 | 133 | for line_data in formatted_text: |
@@ -336,13 +325,48 @@ def _is_header_footer(self, text: str) -> bool: |
336 | 325 | 'security level' in text |
337 | 326 | ) |
338 | 327 |
|
| 328 | + def _has_meaningful_text_content(self, control: Dict) -> bool: |
| 329 | + """Check if a control has meaningful text content in its text attribute.""" |
| 330 | + text = control.get("text", "").strip() |
| 331 | + |
| 332 | + # Consider control as having meaningful content if: |
| 333 | + # 1. It has non-empty text content |
| 334 | + # 2. The text is not just placeholder values like "TODO" or empty strings |
| 335 | + if not text: |
| 336 | + return False |
| 337 | + |
| 338 | + # Check for common placeholder values that should be considered "empty" |
| 339 | + placeholder_values = ["TODO", "TBD", "N/A", "", " "] |
| 340 | + if text in placeholder_values: |
| 341 | + return False |
| 342 | + |
| 343 | + # Must have at least some meaningful content (more than just whitespace) |
| 344 | + if len(text.strip()) < 5: # Arbitrary minimum length for meaningful text |
| 345 | + return False |
| 346 | + |
| 347 | + return True |
| 348 | + |
339 | 349 | def create_nist_catalog_json(self) -> Dict: |
340 | 350 | """Create the full NIST-style catalog JSON structure.""" |
341 | | - # Clean up controls - remove temporary attributes |
| 351 | + # Clean up controls - remove temporary attributes and filter out controls with no text |
342 | 352 | cleaned_controls = [] |
| 353 | + filtered_count = 0 |
| 354 | + |
343 | 355 | for control in self.controls: |
344 | | - cleaned_control = {k: v for k, v in control.items() if k != "_attributes"} |
345 | | - cleaned_controls.append(cleaned_control) |
| 356 | + # Check if control has meaningful text content |
| 357 | + if self._has_meaningful_text_content(control): |
| 358 | + # Remove temporary attributes and add to final list |
| 359 | + cleaned_control = {k: v for k, v in control.items() if k != "_attributes"} |
| 360 | + cleaned_controls.append(cleaned_control) |
| 361 | + else: |
| 362 | + filtered_count += 1 |
| 363 | + if self.debug_mode: |
| 364 | + print(f"Filtered out control {control.get('id', 'unknown')} - no meaningful text content") |
| 365 | + |
| 366 | + # Print filtering summary |
| 367 | + if filtered_count > 0: |
| 368 | + print(f"Filtered out {filtered_count} controls with no or empty text attributes") |
| 369 | + print(f"Remaining controls: {len(cleaned_controls)}") |
346 | 370 |
|
347 | 371 | # Calculate summary statistics |
348 | 372 | summary = self._calculate_summary(cleaned_controls) |
@@ -398,7 +422,7 @@ def save_to_json(self, output_file: str): |
398 | 422 | with open(output_file, 'w', encoding='utf-8') as f: |
399 | 423 | json.dump(catalog, f, indent=2, ensure_ascii=False) |
400 | 424 |
|
401 | | - print(f"\nSuccessfully saved catalog with {len(self.controls)} controls to {output_file}") |
| 425 | + print(f"\nSuccessfully saved catalog with {len(catalog['controls'])} controls to {output_file}") |
402 | 426 | except Exception as e: |
403 | 427 | print(f"Error saving to JSON: {e}") |
404 | 428 |
|
@@ -439,45 +463,28 @@ def debug_page(self, page_num: int): |
439 | 463 | """Debug a specific page to understand formatting.""" |
440 | 464 | try: |
441 | 465 | doc = fitz.open(self.pdf_path) |
442 | | - |
443 | | - if page_num > len(doc): |
444 | | - print(f"Page {page_num} does not exist. Document has {len(doc)} pages.") |
| 466 | + if page_num < 1 or page_num > len(doc): |
| 467 | + print(f"Invalid page number. Document has {len(doc)} pages.") |
445 | 468 | return |
446 | 469 |
|
447 | 470 | page = doc[page_num - 1] # Convert to 0-based index |
448 | | - text_dict = page.get_text("dict") |
449 | | - |
450 | 471 | print(f"\n=== DEBUG PAGE {page_num} ===") |
451 | 472 |
|
452 | | - for block_num, block in enumerate(text_dict.get("blocks", [])): |
453 | | - if block.get("type") == 0: # Text block |
454 | | - print(f"\nBlock {block_num}:") |
455 | | - for line_num, line in enumerate(block.get("lines", [])): |
456 | | - line_text = "" |
457 | | - for span in line.get("spans", []): |
458 | | - line_text += span.get("text", "") |
459 | | - |
460 | | - if line_text.strip(): |
461 | | - print(f" Line {line_num}: '{line_text.strip()}'") |
462 | | - |
463 | | - # Check for control patterns |
464 | | - if re.match(r'^[A-Z]{2}-\d{1,2}[,\s]', line_text.strip()): |
465 | | - print(f" *** POTENTIAL CONTROL ID ***") |
466 | | - |
467 | | - # Check if potential enhancement |
468 | | - if (line_text.strip().startswith('●') or line_text.strip().startswith('•')): |
469 | | - enhancement_match = re.search(r'([A-Z]{2}-\d{1,2}\(\d+\))', line_text) |
470 | | - if enhancement_match: |
471 | | - print(f" *** POTENTIAL ENHANCEMENT: {enhancement_match.group(1)} ***") |
472 | | - |
473 | | - for span in line.get("spans", []): |
474 | | - text = span.get("text", "") |
475 | | - flags = span.get("flags", 0) |
476 | | - font = span.get("font", "") |
477 | | - if text.strip(): |
478 | | - bold = bool(flags & 16) |
479 | | - underline = bool(flags & 2) |
480 | | - print(f" Span: '{text}' | Font: {font} | Bold: {bold} | Underline: {underline}") |
| 473 | + formatted_text = self._get_formatted_text(page) |
| 474 | + |
| 475 | + print(f"Found {len(formatted_text)} text lines:") |
| 476 | + for i, line_data in enumerate(formatted_text): |
| 477 | + line_text = line_data["text"] |
| 478 | + formats = line_data["formats"] |
| 479 | + is_bold = any(fmt.get("bold", False) for fmt in formats) |
| 480 | + |
| 481 | + print(f"{i+1:3d}: {'[B]' if is_bold else ' '} {line_text[:80]}") |
| 482 | + |
| 483 | + # Show detailed format info for first few lines |
| 484 | + if i < 5: |
| 485 | + for fmt in formats: |
| 486 | + if fmt["text"].strip(): |
| 487 | + print(f" Format: '{fmt['text']}' - Bold: {fmt['bold']}, Size: {fmt['size']}") |
481 | 488 |
|
482 | 489 | doc.close() |
483 | 490 |
|
|
0 commit comments