STRATA — DA1: Data Analysis I

from typing import Any, Dict, List

def extract_identifiers_dom(xml_str: str) -> List[Dict[str, Any]]:
    """Extract identifier rows using a DOM-style approach (ElementTree/XPath-style finds).
  
    Parameters:
    - xml_str: srcML XML document as string
    
    Returns:
    - List of identifier dicts, each with keys:
      - name (str): identifier text
      - kind (str): one of 'function', 'parameter', 'variable', 'class'
      - convention (str): naming convention detected
      - length (int): character count
      - n_tokens (int): token count after splitting
      - scope (str): one of 'global', 'local', 'parameter'
    
    Behavior:
    - Parse the entire XML tree with ElementTree
    - Use `.iter()` or `.findall()` to locate function/class/parameter/decl nodes
    - For each <name> node, classify its context (function name vs parameter name vs variable)
    - Return one row per identifier found
    
    Examples:
    >>> xml = '<unit><function><name>process</name></function></unit>'
    >>> ids = extract_identifiers_dom(xml)
    >>> ids[0]['name']
    'process'
    >>> ids[0]['kind']
    'function'
    
    Implementation hints:
    - Use `ET.fromstring(xml_str)` to parse
    - Namespace-aware: strip namespace prefixes with helper (e.g., tag.rsplit('}', 1)[1])
    - Iterate over functions first, then parameters within, then local variables
    - Check parent/ancestor tags to determine context (function vs class vs global)
    """
    # TODO: Implement
    pass


def extract_identifiers_sax(xml_str: str) -> List[Dict[str, Any]]:
    """Extract identifier rows using a SAX event parser.
     
    Parameters:
    - xml_str: srcML XML document as string
    
    Returns:
    - List of identifier dicts with same schema as extract_identifiers_dom
    
    Behavior:
    - Use xml.sax.parse with a custom ContentHandler
    - Maintain a stack of open tags to track context (are we inside <function>? <parameter>?)
    - On endElement for <name>, inspect stack to classify identifier kind/scope
    - Accumulate characters() events into a buffer for each element's text
    
    Examples:
    >>> xml = '<unit><function><name>getData</name></function></unit>'
    >>> ids = extract_identifiers_sax(xml)
    >>> ids[0]['convention']
    'camelCase'
    
    Implementation hints:
    - Subclass xml.sax.handler.ContentHandler
    - Use self.stack = [] to track tag ancestry
    - On startElement, push tag; on endElement, pop
    - Set parser.setFeature(feature_namespaces, True) for namespace support
    - Check stack[-2] or 'function' in stack to determine identifier context
    """
    # TODO: Implement
    pass


# ============================================================================
# NOTE: You only need to implement ONE of the above extraction functions!
# Choose DOM, SAX, or XPath based on your preference and learning goals.
# ============================================================================


def aggregate_identifier_features(identifiers: List[Dict[str, Any]]) -> Dict[str, Any]:
    """Compute file-level aggregate metrics from identifier rows.
    
    Parameters:
    - identifiers: list of identifier dicts from extract_identifiers_dom/sax
    
    Returns:
    - dict with keys:
      - n_identifiers (int): total count
      - avg_identifier_length (float): mean character length
      - avg_tokens_per_identifier (float): mean tokens per name
      - vocab_size (int): unique normalized tokens
      - vocab_diversity (float): unique tokens / total tokens (0.0 to 1.0)
      - pct_snake_case (float): fraction using snake_case
      - pct_camel_case (float): fraction using camelCase
      - pct_pascal_case (float): fraction using PascalCase
    
    Behavior:
    - Return all metrics as 0/0.0 if identifiers list is empty
    - Compute means with simple arithmetic (sum / count)
    - Vocabulary = set of all unique tokens (after lowercasing and splitting)
    - Diversity = len(vocab) / total_token_count (avoid division by zero)
    
    Examples:
    >>> ids = [{'name': 'getUser', 'convention': 'camelCase', 'length': 7, 'tokens': ['get', 'user']}]
    >>> agg = aggregate_identifier_features(ids)
    >>> agg['n_identifiers']
    1
    >>> agg['pct_camel_case']
    1.0
    
    Implementation hints:
    - Use sum() and len() for averages
    - Build vocab with set comprehension: {token for row in identifiers for token in row['tokens']}
    - Count conventions with list comprehension and sum(1 for ...)
    """
    # TODO: Implement
    pass


def build_file_identifier_dataset(xml_by_file: Dict[str, str], parser: str = "dom") -> List[Dict[str, Any]]:
    """Build file-level dataset rows from {file_path: xml_str}.

    Parameters:
    - xml_by_file: dict mapping file paths to srcML XML strings
    - parser: either 'dom' or 'sax' (default 'dom')
    
    Returns:
    - List of dicts, one per file, with keys:
      - file_path (str)
      - n_identifiers (int)
      - avg_identifier_length (float)
      - ... (all metrics from aggregate_identifier_features)
    
    Behavior:
    - Raise ValueError if parser is not 'dom' or 'sax'
    - Process files in sorted order (for reproducibility)
    - For each file: extract identifiers → aggregate → append to output
    
    Examples:
    >>> xml_map = {'a.py': '<unit>...</unit>', 'b.py': '<unit>...</unit>'}
    >>> dataset = build_file_identifier_dataset(xml_map, parser='dom')
    >>> len(dataset)
    2
    >>> dataset[0]['file_path']
    'a.py'
    
    Implementation hints:
    - Normalize parser string: parser.lower().strip()
    - Use sorted(xml_by_file.keys()) for deterministic iteration
    - Call extract_identifiers_dom or extract_identifiers_sax based on parser choice
    - Merge file_path with aggregate dict: {'file_path': path, **agg}
    """
    # TODO: Implement
    pass

import xml.etree.ElementTree as ET

root = ET.fromstring(xml_str)

# Example XPath-style searches (ElementTree subset)
for fn in root.findall('.//function'):
    name_node = fn.find('./name')
    if name_node is not None and (name_node.text or '').strip():
        fn_name = name_node.text.strip()

for param_name in root.findall('.//parameter//name'):
    token = (param_name.text or '').strip()

import io
import xml.sax
from xml.sax.handler import ContentHandler


class IdentifierHandler(ContentHandler):
    def __init__(self):
        super().__init__()
        self.stack = []
        self.buffer = []
        self.rows = []

    def startElement(self, name, attrs):
        self.stack.append(name)
        self.buffer = []

    def characters(self, content):
        self.buffer.append(content)

    def endElement(self, name):
        text = ''.join(self.buffer).strip()
        # Decide whether this <name> corresponds to function/parameter/decl/class
        # based on self.stack context, then append to self.rows
        self.stack.pop()
        self.buffer = []


handler = IdentifierHandler()
xml.sax.parse(io.StringIO(xml_str), handler)
rows = handler.rows

from lxml import etree

root = etree.fromstring(xml_str.encode('utf-8'))

# Full XPath 1.0 support with axes and predicates
# Function names: first <name> child of any <function>
for name_node in root.xpath('//function/name[1]'):
    text = (name_node.text or '').strip()

# Local variables: <decl_stmt> WITH <function> ancestor
for name_node in root.xpath('//decl_stmt[ancestor::function]//decl/name[1]'):
    text = (name_node.text or '').strip()

# Global variables: <decl_stmt> WITHOUT <function> ancestor
for name_node in root.xpath('//decl_stmt[not(ancestor::function)]//decl/name[1]'):
    text = (name_node.text or '').strip()

pip install lxml

from src import srcml_runner
from src import DA1_identifiers

xml = srcml_runner.run_srcml_on_repo_file('.', 'src/qual_clean.py', commit='HEAD')
ids = DA1_identifiers.extract_identifiers_dom(xml)
summary = DA1_identifiers.aggregate_identifier_features(ids)
print(summary)

pytest test/test_DA1_identifiers.py -v