GITENV file updated 20

cgpenv/Lib/site-packages/spacy/pipeline/__init__.py

+from .attributeruler import AttributeRuler
+from .dep_parser import DependencyParser
+from .entity_linker import EntityLinker
+from .ner import EntityRecognizer
+from .entityruler import EntityRuler
+from .lemmatizer import Lemmatizer
+from .morphologizer import Morphologizer
+from .pipe import Pipe
+from .trainable_pipe import TrainablePipe
+from .senter import SentenceRecognizer
+from .sentencizer import Sentencizer
+from .tagger import Tagger
+from .textcat import TextCategorizer
+from .spancat import SpanCategorizer
+from .textcat_multilabel import MultiLabel_TextCategorizer
+from .tok2vec import Tok2Vec
+from .functions import merge_entities, merge_noun_chunks, merge_subtokens
+
+__all__ = [
+    "AttributeRuler",
+    "DependencyParser",
+    "EntityLinker",
+    "EntityRecognizer",
+    "EntityRuler",
+    "Morphologizer",
+    "Lemmatizer",
+    "MultiLabel_TextCategorizer",
+    "Pipe",
+    "SentenceRecognizer",
+    "Sentencizer",
+    "SpanCategorizer",
+    "Tagger",
+    "TextCategorizer",
+    "Tok2Vec",
+    "TrainablePipe",
+    "merge_entities",
+    "merge_noun_chunks",
+    "merge_subtokens",
+]

cgpenv/Lib/site-packages/spacy/pipeline/attributeruler.py

+from typing import List, Dict, Union, Iterable, Any, Optional, Callable
+from typing import Tuple
+import srsly
+from pathlib import Path
+
+from .pipe import Pipe
+from ..errors import Errors
+from ..training import Example
+from ..language import Language
+from ..matcher import Matcher
+from ..scorer import Scorer
+from ..symbols import IDS
+from ..tokens import Doc, Span
+from ..tokens._retokenize import normalize_token_attrs, set_token_attrs
+from ..vocab import Vocab
+from ..util import SimpleFrozenList, registry
+from .. import util
+
+
+MatcherPatternType = List[Dict[Union[int, str], Any]]
+AttributeRulerPatternType = Dict[str, Union[MatcherPatternType, Dict, int]]
+TagMapType = Dict[str, Dict[Union[int, str], Union[int, str]]]
+MorphRulesType = Dict[str, Dict[str, Dict[Union[int, str], Union[int, str]]]]
+
+
+@Language.factory(
+    "attribute_ruler",
+    default_config={
+        "validate": False,
+        "scorer": {"@scorers": "spacy.attribute_ruler_scorer.v1"},
+    },
+)
+def make_attribute_ruler(
+    nlp: Language, name: str, validate: bool, scorer: Optional[Callable]
+):
+    return AttributeRuler(nlp.vocab, name, validate=validate, scorer=scorer)
+
+
+def attribute_ruler_score(examples: Iterable[Example], **kwargs) -> Dict[str, Any]:
+    def morph_key_getter(token, attr):
+        return getattr(token, attr).key
+
+    results = {}
+    results.update(Scorer.score_token_attr(examples, "tag", **kwargs))
+    results.update(Scorer.score_token_attr(examples, "pos", **kwargs))
+    results.update(
+        Scorer.score_token_attr(examples, "morph", getter=morph_key_getter, **kwargs)
+    )
+    results.update(
+        Scorer.score_token_attr_per_feat(
+            examples, "morph", getter=morph_key_getter, **kwargs
+        )
+    )
+    results.update(Scorer.score_token_attr(examples, "lemma", **kwargs))
+    return results
+
+
+@registry.scorers("spacy.attribute_ruler_scorer.v1")
+def make_attribute_ruler_scorer():
+    return attribute_ruler_score
+
+
+class AttributeRuler(Pipe):
+    """Set token-level attributes for tokens matched by Matcher patterns.
+    Additionally supports importing patterns from tag maps and morph rules.
+
+    DOCS: https://spacy.io/api/attributeruler
+    """
+
+    def __init__(
+        self,
+        vocab: Vocab,
+        name: str = "attribute_ruler",
+        *,
+        validate: bool = False,
+        scorer: Optional[Callable] = attribute_ruler_score,
+    ) -> None:
+        """Create the AttributeRuler. After creation, you can add patterns
+        with the `.initialize()` or `.add_patterns()` methods, or load patterns
+        with `.from_bytes()` or `.from_disk()`. Loading patterns will remove
+        any patterns you've added previously.
+
+        vocab (Vocab): The vocab.
+        name (str): The pipe name. Defaults to "attribute_ruler".
+        scorer (Optional[Callable]): The scoring method. Defaults to
+            Scorer.score_token_attr for the attributes "tag", "pos", "morph" and
+            "lemma" and Scorer.score_token_attr_per_feat for the attribute
+            "morph".
+
+        RETURNS (AttributeRuler): The AttributeRuler component.
+
+        DOCS: https://spacy.io/api/attributeruler#init
+        """
+        self.name = name
+        self.vocab = vocab
+        self.matcher = Matcher(self.vocab, validate=validate)
+        self.validate = validate
+        self.attrs: List[Dict] = []
+        self._attrs_unnormed: List[Dict] = []  # store for reference
+        self.indices: List[int] = []
+        self.scorer = scorer
+
+    def clear(self) -> None:
+        """Reset all patterns."""
+        self.matcher = Matcher(self.vocab, validate=self.validate)
+        self.attrs = []
+        self._attrs_unnormed = []
+        self.indices = []
+
+    def initialize(
+        self,
+        get_examples: Optional[Callable[[], Iterable[Example]]],
+        *,
+        nlp: Optional[Language] = None,
+        patterns: Optional[Iterable[AttributeRulerPatternType]] = None,
+        tag_map: Optional[TagMapType] = None,
+        morph_rules: Optional[MorphRulesType] = None,
+    ) -> None:
+        """Initialize the attribute ruler by adding zero or more patterns.
+
+        Rules can be specified as a sequence of dicts using the `patterns`
+        keyword argument. You can also provide rules using the "tag map" or
+        "morph rules" formats supported by spaCy prior to v3.
+        """
+        self.clear()
+        if patterns:
+            self.add_patterns(patterns)
+        if tag_map:
+            self.load_from_tag_map(tag_map)
+        if morph_rules:
+            self.load_from_morph_rules(morph_rules)
+
+    def __call__(self, doc: Doc) -> Doc:
+        """Apply the AttributeRuler to a Doc and set all attribute exceptions.
+
+        doc (Doc): The document to process.
+        RETURNS (Doc): The processed Doc.
+
+        DOCS: https://spacy.io/api/attributeruler#call
+        """
+        error_handler = self.get_error_handler()
+        try:
+            matches = self.match(doc)
+            self.set_annotations(doc, matches)
+            return doc
+        except Exception as e:
+            return error_handler(self.name, self, [doc], e)
+
+    def match(self, doc: Doc):
+        matches = self.matcher(doc, allow_missing=True, as_spans=False)
+        # Sort by the attribute ID, so that later rules have precedence
+        matches = [
+            (int(self.vocab.strings[m_id]), m_id, s, e) for m_id, s, e in matches  # type: ignore
+        ]
+        matches.sort()
+        return matches
+
+    def set_annotations(self, doc, matches):
+        """Modify the document in place"""
+        for attr_id, match_id, start, end in matches:
+            span = Span(doc, start, end, label=match_id)
+            attrs = self.attrs[attr_id]
+            index = self.indices[attr_id]
+            try:
+                # The index can be negative, which makes it annoying to do
+                # the boundscheck. Let Span do it instead.
+                token = span[index]  # noqa: F841
+            except IndexError:
+                # The original exception is just our conditional logic, so we
+                # raise from.
+                raise ValueError(
+                    Errors.E1001.format(
+                        patterns=self.matcher.get(span.label),
+                        span=[t.text for t in span],
+                        index=index,
+                    )
+                ) from None
+            set_token_attrs(span[index], attrs)
+
+    def load_from_tag_map(
+        self, tag_map: Dict[str, Dict[Union[int, str], Union[int, str]]]
+    ) -> None:
+        """Load attribute ruler patterns from a tag map.
+
+        tag_map (dict): The tag map that maps fine-grained tags to
+            coarse-grained tags and morphological features.
+
+        DOCS: https://spacy.io/api/attributeruler#load_from_morph_rules
+        """
+        for tag, attrs in tag_map.items():
+            pattern = [{"TAG": tag}]
+            attrs, morph_attrs = _split_morph_attrs(attrs)
+            if "MORPH" not in attrs:
+                morph = self.vocab.morphology.add(morph_attrs)
+                attrs["MORPH"] = self.vocab.strings[morph]
+            else:
+                morph = self.vocab.morphology.add(attrs["MORPH"])
+                attrs["MORPH"] = self.vocab.strings[morph]
+            self.add([pattern], attrs)  # type: ignore[list-item]
+
+    def load_from_morph_rules(
+        self, morph_rules: Dict[str, Dict[str, Dict[Union[int, str], Union[int, str]]]]
+    ) -> None:
+        """Load attribute ruler patterns from morph rules.
+
+        morph_rules (dict): The morph rules that map token text and
+            fine-grained tags to coarse-grained tags, lemmas and morphological
+            features.
+
+        DOCS: https://spacy.io/api/attributeruler#load_from_morph_rules
+        """
+        for tag in morph_rules:
+            for word in morph_rules[tag]:
+                pattern = [{"ORTH": word, "TAG": tag}]
+                attrs = morph_rules[tag][word]
+                attrs, morph_attrs = _split_morph_attrs(attrs)
+                if "MORPH" in attrs:
+                    morph = self.vocab.morphology.add(attrs["MORPH"])
+                    attrs["MORPH"] = self.vocab.strings[morph]
+                elif morph_attrs:
+                    morph = self.vocab.morphology.add(morph_attrs)
+                    attrs["MORPH"] = self.vocab.strings[morph]
+                self.add([pattern], attrs)  # type: ignore[list-item]
+
+    def add(
+        self, patterns: Iterable[MatcherPatternType], attrs: Dict, index: int = 0
+    ) -> None:
+        """Add Matcher patterns for tokens that should be modified with the
+        provided attributes. The token at the specified index within the
+        matched span will be assigned the attributes.
+
+        patterns (Iterable[List[Dict]]): A list of Matcher patterns.
+        attrs (Dict): The attributes to assign to the target token in the
+            matched span.
+        index (int): The index of the token in the matched span to modify. May
+            be negative to index from the end of the span. Defaults to 0.
+
+        DOCS: https://spacy.io/api/attributeruler#add
+        """
+        # We need to make a string here, because otherwise the ID we pass back
+        # will be interpreted as the hash of a string, rather than an ordinal.
+        key = str(len(self.attrs))
+        self.matcher.add(self.vocab.strings.add(key), patterns)  # type: ignore[arg-type]
+        self._attrs_unnormed.append(attrs)
+        attrs = normalize_token_attrs(self.vocab, attrs)
+        self.attrs.append(attrs)
+        self.indices.append(index)
+
+    def add_patterns(self, patterns: Iterable[AttributeRulerPatternType]) -> None:
+        """Add patterns from a list of pattern dicts with the keys as the
+        arguments to AttributeRuler.add.
+        patterns (Iterable[dict]): A list of pattern dicts with the keys
+            as the arguments to AttributeRuler.add (patterns/attrs/index) to
+            add as patterns.
+
+        DOCS: https://spacy.io/api/attributeruler#add_patterns
+        """
+        for p in patterns:
+            self.add(**p)  # type: ignore[arg-type]
+
+    @property
+    def patterns(self) -> List[AttributeRulerPatternType]:
+        """All the added patterns."""
+        all_patterns = []
+        for i in range(len(self.attrs)):
+            p = {}
+            p["patterns"] = self.matcher.get(str(i))[1]
+            p["attrs"] = self._attrs_unnormed[i]  # type: ignore
+            p["index"] = self.indices[i]  # type: ignore
+            all_patterns.append(p)
+        return all_patterns  # type: ignore[return-value]
+
+    def to_bytes(self, exclude: Iterable[str] = SimpleFrozenList()) -> bytes:
+        """Serialize the AttributeRuler to a bytestring.
+
+        exclude (Iterable[str]): String names of serialization fields to exclude.
+        RETURNS (bytes): The serialized object.
+
+        DOCS: https://spacy.io/api/attributeruler#to_bytes
+        """
+        serialize = {}
+        serialize["vocab"] = lambda: self.vocab.to_bytes(exclude=exclude)
+        serialize["patterns"] = lambda: srsly.msgpack_dumps(self.patterns)
+        return util.to_bytes(serialize, exclude)
+
+    def from_bytes(
+        self, bytes_data: bytes, exclude: Iterable[str] = SimpleFrozenList()
+    ) -> "AttributeRuler":
+        """Load the AttributeRuler from a bytestring.
+
+        bytes_data (bytes): The data to load.
+        exclude (Iterable[str]): String names of serialization fields to exclude.
+        returns (AttributeRuler): The loaded object.
+
+        DOCS: https://spacy.io/api/attributeruler#from_bytes
+        """
+
+        def load_patterns(b):
+            self.add_patterns(srsly.msgpack_loads(b))
+
+        deserialize = {
+            "vocab": lambda b: self.vocab.from_bytes(b, exclude=exclude),
+            "patterns": load_patterns,
+        }
+        util.from_bytes(bytes_data, deserialize, exclude)
+        return self
+
+    def to_disk(
+        self, path: Union[Path, str], exclude: Iterable[str] = SimpleFrozenList()
+    ) -> None:
+        """Serialize the AttributeRuler to disk.
+
+        path (Union[Path, str]): A path to a directory.
+        exclude (Iterable[str]): String names of serialization fields to exclude.
+
+        DOCS: https://spacy.io/api/attributeruler#to_disk
+        """
+        serialize = {
+            "vocab": lambda p: self.vocab.to_disk(p, exclude=exclude),
+            "patterns": lambda p: srsly.write_msgpack(p, self.patterns),
+        }
+        util.to_disk(path, serialize, exclude)
+
+    def from_disk(
+        self, path: Union[Path, str], exclude: Iterable[str] = SimpleFrozenList()
+    ) -> "AttributeRuler":
+        """Load the AttributeRuler from disk.
+
+        path (Union[Path, str]): A path to a directory.
+        exclude (Iterable[str]): String names of serialization fields to exclude.
+        RETURNS (AttributeRuler): The loaded object.
+
+        DOCS: https://spacy.io/api/attributeruler#from_disk
+        """
+
+        def load_patterns(p):
+            self.add_patterns(srsly.read_msgpack(p))
+
+        deserialize = {
+            "vocab": lambda p: self.vocab.from_disk(p, exclude=exclude),
+            "patterns": load_patterns,
+        }
+        util.from_disk(path, deserialize, exclude)
+        return self
+
+
+def _split_morph_attrs(attrs: dict) -> Tuple[dict, dict]:
+    """Split entries from a tag map or morph rules dict into to two dicts, one
+    with the token-level features (POS, LEMMA) and one with the remaining
+    features, which are presumed to be individual MORPH features."""
+    other_attrs = {}
+    morph_attrs = {}
+    for k, v in attrs.items():
+        if k in "_" or k in IDS.keys() or k in IDS.values():
+            other_attrs[k] = v
+        else:
+            morph_attrs[k] = v
+    return other_attrs, morph_attrs

cgpenv/Lib/site-packages/spacy/pipeline/dep_parser.pyx

+# cython: infer_types=True, profile=True, binding=True
+from collections import defaultdict
+from typing import Optional, Iterable, Callable
+from thinc.api import Model, Config
+
+from ._parser_internals.transition_system import TransitionSystem
+from .transition_parser cimport Parser
+from ._parser_internals.arc_eager cimport ArcEager
+
+from .functions import merge_subtokens
+from ..language import Language
+from ._parser_internals import nonproj
+from ._parser_internals.nonproj import DELIMITER
+from ..scorer import Scorer
+from ..util import registry
+
+
+default_model_config = """
+[model]
+@architectures = "spacy.TransitionBasedParser.v2"
+state_type = "parser"
+extra_state_tokens = false
+hidden_width = 64
+maxout_pieces = 2
+use_upper = true
+
+[model.tok2vec]
+@architectures = "spacy.HashEmbedCNN.v2"
+pretrained_vectors = null
+width = 96
+depth = 4
+embed_size = 2000
+window_size = 1
+maxout_pieces = 3
+subword_features = true
+"""
+DEFAULT_PARSER_MODEL = Config().from_str(default_model_config)["model"]
+
+
+@Language.factory(
+    "parser",
+    assigns=["token.dep", "token.head", "token.is_sent_start", "doc.sents"],
+    default_config={
+        "moves": None,
+        "update_with_oracle_cut_size": 100,
+        "learn_tokens": False,
+        "min_action_freq": 30,
+        "model": DEFAULT_PARSER_MODEL,
+        "scorer": {"@scorers": "spacy.parser_scorer.v1"},
+    },
+    default_score_weights={
+        "dep_uas": 0.5,
+        "dep_las": 0.5,
+        "dep_las_per_type": None,
+        "sents_p": None,
+        "sents_r": None,
+        "sents_f": 0.0,
+    },
+)
+def make_parser(
+    nlp: Language,
+    name: str,
+    model: Model,
+    moves: Optional[TransitionSystem],
+    update_with_oracle_cut_size: int,
+    learn_tokens: bool,
+    min_action_freq: int,
+    scorer: Optional[Callable],
+):
+    """Create a transition-based DependencyParser component. The dependency parser
+    jointly learns sentence segmentation and labelled dependency parsing, and can
+    optionally learn to merge tokens that had been over-segmented by the tokenizer.
+
+    The parser uses a variant of the non-monotonic arc-eager transition-system
+    described by Honnibal and Johnson (2014), with the addition of a "break"
+    transition to perform the sentence segmentation. Nivre's pseudo-projective
+    dependency transformation is used to allow the parser to predict
+    non-projective parses.
+
+    The parser is trained using an imitation learning objective. The parser follows
+    the actions predicted by the current weights, and at each state, determines
+    which actions are compatible with the optimal parse that could be reached
+    from the current state. The weights such that the scores assigned to the
+    set of optimal actions is increased, while scores assigned to other
+    actions are decreased. Note that more than one action may be optimal for
+    a given state.
+
+    model (Model): The model for the transition-based parser. The model needs
+        to have a specific substructure of named components --- see the
+        spacy.ml.tb_framework.TransitionModel for details.
+    moves (Optional[TransitionSystem]): This defines how the parse-state is created,
+        updated and evaluated. If 'moves' is None, a new instance is
+        created with `self.TransitionSystem()`. Defaults to `None`.
+    update_with_oracle_cut_size (int): During training, cut long sequences into
+        shorter segments by creating intermediate states based on the gold-standard
+        history. The model is not very sensitive to this parameter, so you usually
+        won't need to change it. 100 is a good default.
+    learn_tokens (bool): Whether to learn to merge subtokens that are split
+        relative to the gold standard. Experimental.
+    min_action_freq (int): The minimum frequency of labelled actions to retain.
+        Rarer labelled actions have their label backed-off to "dep". While this
+        primarily affects the label accuracy, it can also affect the attachment
+        structure, as the labels are used to represent the pseudo-projectivity
+        transformation.
+    scorer (Optional[Callable]): The scoring method.
+    """
+    return DependencyParser(
+        nlp.vocab,
+        model,
+        name,
+        moves=moves,
+        update_with_oracle_cut_size=update_with_oracle_cut_size,
+        multitasks=[],
+        learn_tokens=learn_tokens,
+        min_action_freq=min_action_freq,
+        beam_width=1,
+        beam_density=0.0,
+        beam_update_prob=0.0,
+        # At some point in the future we can try to implement support for
+        # partial annotations, perhaps only in the beam objective.
+        incorrect_spans_key=None,
+        scorer=scorer,
+    )
+
+@Language.factory(
+    "beam_parser",
+    assigns=["token.dep", "token.head", "token.is_sent_start", "doc.sents"],
+    default_config={
+        "beam_width": 8,
+        "beam_density": 0.01,
+        "beam_update_prob": 0.5,
+        "moves": None,
+        "update_with_oracle_cut_size": 100,
+        "learn_tokens": False,
+        "min_action_freq": 30,
+        "model": DEFAULT_PARSER_MODEL,
+        "scorer": {"@scorers": "spacy.parser_scorer.v1"},
+    },
+    default_score_weights={
+        "dep_uas": 0.5,
+        "dep_las": 0.5,
+        "dep_las_per_type": None,
+        "sents_p": None,
+        "sents_r": None,
+        "sents_f": 0.0,
+    },
+)
+def make_beam_parser(
+    nlp: Language,
+    name: str,
+    model: Model,
+    moves: Optional[TransitionSystem],
+    update_with_oracle_cut_size: int,
+    learn_tokens: bool,
+    min_action_freq: int,
+    beam_width: int,
+    beam_density: float,
+    beam_update_prob: float,
+    scorer: Optional[Callable],
+):
+    """Create a transition-based DependencyParser component that uses beam-search.
+    The dependency parser jointly learns sentence segmentation and labelled
+    dependency parsing, and can optionally learn to merge tokens that had been
+    over-segmented by the tokenizer.
+
+    The parser uses a variant of the non-monotonic arc-eager transition-system
+    described by Honnibal and Johnson (2014), with the addition of a "break"
+    transition to perform the sentence segmentation. Nivre's pseudo-projective
+    dependency transformation is used to allow the parser to predict
+    non-projective parses.
+
+    The parser is trained using a global objective. That is, it learns to assign
+    probabilities to whole parses.
+
+    model (Model): The model for the transition-based parser. The model needs
+        to have a specific substructure of named components --- see the
+        spacy.ml.tb_framework.TransitionModel for details.
+    moves (Optional[TransitionSystem]): This defines how the parse-state is created,
+        updated and evaluated. If 'moves' is None, a new instance is
+        created with `self.TransitionSystem()`. Defaults to `None`.
+    update_with_oracle_cut_size (int): During training, cut long sequences into
+        shorter segments by creating intermediate states based on the gold-standard
+        history. The model is not very sensitive to this parameter, so you usually
+        won't need to change it. 100 is a good default.
+    beam_width (int): The number of candidate analyses to maintain.
+    beam_density (float): The minimum ratio between the scores of the first and
+        last candidates in the beam. This allows the parser to avoid exploring
+        candidates that are too far behind. This is mostly intended to improve
+        efficiency, but it can also improve accuracy as deeper search is not
+        always better.
+    beam_update_prob (float): The chance of making a beam update, instead of a
+        greedy update. Greedy updates are an approximation for the beam updates,
+        and are faster to compute.
+    learn_tokens (bool): Whether to learn to merge subtokens that are split
+        relative to the gold standard. Experimental.
+    min_action_freq (int): The minimum frequency of labelled actions to retain.
+        Rarer labelled actions have their label backed-off to "dep". While this
+        primarily affects the label accuracy, it can also affect the attachment
+        structure, as the labels are used to represent the pseudo-projectivity
+        transformation.
+    """
+    return DependencyParser(
+        nlp.vocab,
+        model,
+        name,
+        moves=moves,
+        update_with_oracle_cut_size=update_with_oracle_cut_size,
+        beam_width=beam_width,
+        beam_density=beam_density,
+        beam_update_prob=beam_update_prob,
+        multitasks=[],
+        learn_tokens=learn_tokens,
+        min_action_freq=min_action_freq,
+        # At some point in the future we can try to implement support for
+        # partial annotations, perhaps only in the beam objective.
+        incorrect_spans_key=None,
+        scorer=scorer,
+    )
+
+
+def parser_score(examples, **kwargs):
+    """Score a batch of examples.
+
+    examples (Iterable[Example]): The examples to score.
+    RETURNS (Dict[str, Any]): The scores, produced by Scorer.score_spans
+        and Scorer.score_deps.
+
+    DOCS: https://spacy.io/api/dependencyparser#score
+    """
+    def has_sents(doc):
+        return doc.has_annotation("SENT_START")
+
+    def dep_getter(token, attr):
+        dep = getattr(token, attr)
+        dep = token.vocab.strings.as_string(dep).lower()
+        return dep
+    results = {}
+    results.update(Scorer.score_spans(examples, "sents", has_annotation=has_sents, **kwargs))
+    kwargs.setdefault("getter", dep_getter)
+    kwargs.setdefault("ignore_labels", ("p", "punct"))
+    results.update(Scorer.score_deps(examples, "dep", **kwargs))
+    del results["sents_per_type"]
+    return results
+
+
+@registry.scorers("spacy.parser_scorer.v1")
+def make_parser_scorer():
+    return parser_score
+
+
+cdef class DependencyParser(Parser):
+    """Pipeline component for dependency parsing.
+
+    DOCS: https://spacy.io/api/dependencyparser
+    """
+    TransitionSystem = ArcEager
+
+    def __init__(
+        self,
+        vocab,
+        model,
+        name="parser",
+        moves=None,
+        *,
+        update_with_oracle_cut_size=100,
+        min_action_freq=30,
+        learn_tokens=False,
+        beam_width=1,
+        beam_density=0.0,
+        beam_update_prob=0.0,
+        multitasks=tuple(),
+        incorrect_spans_key=None,
+        scorer=parser_score,
+    ):
+        """Create a DependencyParser.
+        """
+        super().__init__(
+            vocab,
+            model,
+            name,
+            moves,
+            update_with_oracle_cut_size=update_with_oracle_cut_size,
+            min_action_freq=min_action_freq,
+            learn_tokens=learn_tokens,
+            beam_width=beam_width,
+            beam_density=beam_density,
+            beam_update_prob=beam_update_prob,
+            multitasks=multitasks,
+            incorrect_spans_key=incorrect_spans_key,
+            scorer=scorer,
+        )
+
+    @property
+    def postprocesses(self):
+        output = [nonproj.deprojectivize]
+        if self.cfg.get("learn_tokens") is True:
+            output.append(merge_subtokens)
+        return tuple(output)
+
+    def add_multitask_objective(self, mt_component):
+        self._multitasks.append(mt_component)
+
+    def init_multitask_objectives(self, get_examples, nlp=None, **cfg):
+        # TODO: transfer self.model.get_ref("tok2vec") to the multitask's model ?
+        for labeller in self._multitasks:
+            labeller.model.set_dim("nO", len(self.labels))
+            if labeller.model.has_ref("output_layer"):
+                labeller.model.get_ref("output_layer").set_dim("nO", len(self.labels))
+            labeller.initialize(get_examples, nlp=nlp)
+
+    @property
+    def labels(self):
+        labels = set()
+        # Get the labels from the model by looking at the available moves
+        for move in self.move_names:
+            if "-" in move:
+                label = move.split("-")[1]
+                if DELIMITER in label:
+                    label = label.split(DELIMITER)[1]
+                labels.add(label)
+        return tuple(sorted(labels))
+
+    def scored_parses(self, beams):
+        """Return two dictionaries with scores for each beam/doc that was processed:
+        one containing (i, head) keys, and another containing (i, label) keys.
+        """
+        head_scores = []
+        label_scores = []
+        for beam in beams:
+            score_head_dict = defaultdict(float)
+            score_label_dict = defaultdict(float)
+            for score, parses in self.moves.get_beam_parses(beam):
+                for head, i, label in parses:
+                    score_head_dict[(i, head)] += score
+                    score_label_dict[(i, label)] += score
+            head_scores.append(score_head_dict)
+            label_scores.append(score_label_dict)
+        return head_scores, label_scores
+
+    def _ensure_labels_are_added(self, docs):
+        # This gives the parser a chance to add labels it's missing for a batch
+        # of documents. However, this isn't desirable for the dependency parser,
+        # because we instead have a label frequency cut-off and back off rare
+        # labels to 'dep'.
+        pass

cgpenv/Lib/site-packages/spacy/pipeline/entity_linker.py

+from typing import Optional, Iterable, Callable, Dict, Union, List, Any
+from thinc.types import Floats2d
+from pathlib import Path
+from itertools import islice
+import srsly
+import random
+from thinc.api import CosineDistance, Model, Optimizer, Config
+from thinc.api import set_dropout_rate
+import warnings
+
+from ..kb import KnowledgeBase, Candidate
+from ..ml import empty_kb
+from ..tokens import Doc, Span
+from .pipe import deserialize_config
+from .trainable_pipe import TrainablePipe
+from ..language import Language
+from ..vocab import Vocab
+from ..training import Example, validate_examples, validate_get_examples
+from ..errors import Errors, Warnings
+from ..util import SimpleFrozenList, registry
+from .. import util
+from ..scorer import Scorer
+
+# See #9050
+BACKWARD_OVERWRITE = True
+
+default_model_config = """
+[model]
+@architectures = "spacy.EntityLinker.v1"
+
+[model.tok2vec]
+@architectures = "spacy.HashEmbedCNN.v2"
+pretrained_vectors = null
+width = 96
+depth = 2
+embed_size = 2000
+window_size = 1
+maxout_pieces = 3
+subword_features = true
+"""
+DEFAULT_NEL_MODEL = Config().from_str(default_model_config)["model"]
+
+
+@Language.factory(
+    "entity_linker",
+    requires=["doc.ents", "doc.sents", "token.ent_iob", "token.ent_type"],
+    assigns=["token.ent_kb_id"],
+    default_config={
+        "model": DEFAULT_NEL_MODEL,
+        "labels_discard": [],
+        "n_sents": 0,
+        "incl_prior": True,
+        "incl_context": True,
+        "entity_vector_length": 64,
+        "get_candidates": {"@misc": "spacy.CandidateGenerator.v1"},
+        "overwrite": True,
+        "scorer": {"@scorers": "spacy.entity_linker_scorer.v1"},
+    },
+    default_score_weights={
+        "nel_micro_f": 1.0,
+        "nel_micro_r": None,
+        "nel_micro_p": None,
+    },
+)
+def make_entity_linker(
+    nlp: Language,
+    name: str,
+    model: Model,
+    *,
+    labels_discard: Iterable[str],
+    n_sents: int,
+    incl_prior: bool,
+    incl_context: bool,
+    entity_vector_length: int,
+    get_candidates: Callable[[KnowledgeBase, Span], Iterable[Candidate]],
+    overwrite: bool,
+    scorer: Optional[Callable],
+):
+    """Construct an EntityLinker component.
+
+    model (Model[List[Doc], Floats2d]): A model that learns document vector
+        representations. Given a batch of Doc objects, it should return a single
+        array, with one row per item in the batch.
+    labels_discard (Iterable[str]): NER labels that will automatically get a "NIL" prediction.
+    n_sents (int): The number of neighbouring sentences to take into account.
+    incl_prior (bool): Whether or not to include prior probabilities from the KB in the model.
+    incl_context (bool): Whether or not to include the local context in the model.
+    entity_vector_length (int): Size of encoding vectors in the KB.
+    get_candidates (Callable[[KnowledgeBase, "Span"], Iterable[Candidate]]): Function that
+        produces a list of candidates, given a certain knowledge base and a textual mention.
+    scorer (Optional[Callable]): The scoring method.
+    """
+    return EntityLinker(
+        nlp.vocab,
+        model,
+        name,
+        labels_discard=labels_discard,
+        n_sents=n_sents,
+        incl_prior=incl_prior,
+        incl_context=incl_context,
+        entity_vector_length=entity_vector_length,
+        get_candidates=get_candidates,
+        overwrite=overwrite,
+        scorer=scorer,
+    )
+
+
+def entity_linker_score(examples, **kwargs):
+    return Scorer.score_links(examples, negative_labels=[EntityLinker.NIL], **kwargs)
+
+
+@registry.scorers("spacy.entity_linker_scorer.v1")
+def make_entity_linker_scorer():
+    return entity_linker_score
+
+
+class EntityLinker(TrainablePipe):
+    """Pipeline component for named entity linking.
+
+    DOCS: https://spacy.io/api/entitylinker
+    """
+
+    NIL = "NIL"  # string used to refer to a non-existing link
+
+    def __init__(
+        self,
+        vocab: Vocab,
+        model: Model,
+        name: str = "entity_linker",
+        *,
+        labels_discard: Iterable[str],
+        n_sents: int,
+        incl_prior: bool,
+        incl_context: bool,
+        entity_vector_length: int,
+        get_candidates: Callable[[KnowledgeBase, Span], Iterable[Candidate]],
+        overwrite: bool = BACKWARD_OVERWRITE,
+        scorer: Optional[Callable] = entity_linker_score,
+    ) -> None:
+        """Initialize an entity linker.
+
+        vocab (Vocab): The shared vocabulary.
+        model (thinc.api.Model): The Thinc Model powering the pipeline component.
+        name (str): The component instance name, used to add entries to the
+            losses during training.
+        labels_discard (Iterable[str]): NER labels that will automatically get a "NIL" prediction.
+        n_sents (int): The number of neighbouring sentences to take into account.
+        incl_prior (bool): Whether or not to include prior probabilities from the KB in the model.
+        incl_context (bool): Whether or not to include the local context in the model.
+        entity_vector_length (int): Size of encoding vectors in the KB.
+        get_candidates (Callable[[KnowledgeBase, Span], Iterable[Candidate]]): Function that
+            produces a list of candidates, given a certain knowledge base and a textual mention.
+        scorer (Optional[Callable]): The scoring method. Defaults to
+            Scorer.score_links.
+
+        DOCS: https://spacy.io/api/entitylinker#init
+        """
+        self.vocab = vocab
+        self.model = model
+        self.name = name
+        self.labels_discard = list(labels_discard)
+        self.n_sents = n_sents
+        self.incl_prior = incl_prior
+        self.incl_context = incl_context
+        self.get_candidates = get_candidates
+        self.cfg: Dict[str, Any] = {"overwrite": overwrite}
+        self.distance = CosineDistance(normalize=False)
+        # how many neighbour sentences to take into account
+        # create an empty KB by default. If you want to load a predefined one, specify it in 'initialize'.
+        self.kb = empty_kb(entity_vector_length)(self.vocab)
+        self.scorer = scorer
+
+    def set_kb(self, kb_loader: Callable[[Vocab], KnowledgeBase]):
+        """Define the KB of this pipe by providing a function that will
+        create it using this object's vocab."""
+        if not callable(kb_loader):
+            raise ValueError(Errors.E885.format(arg_type=type(kb_loader)))
+
+        self.kb = kb_loader(self.vocab)
+
+    def validate_kb(self) -> None:
+        # Raise an error if the knowledge base is not initialized.
+        if self.kb is None:
+            raise ValueError(Errors.E1018.format(name=self.name))
+        if len(self.kb) == 0:
+            raise ValueError(Errors.E139.format(name=self.name))
+
+    def initialize(
+        self,
+        get_examples: Callable[[], Iterable[Example]],
+        *,
+        nlp: Optional[Language] = None,
+        kb_loader: Optional[Callable[[Vocab], KnowledgeBase]] = None,
+    ):
+        """Initialize the pipe for training, using a representative set
+        of data examples.
+
+        get_examples (Callable[[], Iterable[Example]]): Function that
+            returns a representative sample of gold-standard Example objects.
+        nlp (Language): The current nlp object the component is part of.
+        kb_loader (Callable[[Vocab], KnowledgeBase]): A function that creates a KnowledgeBase from a Vocab instance.
+            Note that providing this argument, will overwrite all data accumulated in the current KB.
+            Use this only when loading a KB as-such from file.
+
+        DOCS: https://spacy.io/api/entitylinker#initialize
+        """
+        validate_get_examples(get_examples, "EntityLinker.initialize")
+        if kb_loader is not None:
+            self.set_kb(kb_loader)
+        self.validate_kb()
+        nO = self.kb.entity_vector_length
+        doc_sample = []
+        vector_sample = []
+        for example in islice(get_examples(), 10):
+            doc_sample.append(example.x)
+            vector_sample.append(self.model.ops.alloc1f(nO))
+        assert len(doc_sample) > 0, Errors.E923.format(name=self.name)
+        assert len(vector_sample) > 0, Errors.E923.format(name=self.name)
+        self.model.initialize(
+            X=doc_sample, Y=self.model.ops.asarray(vector_sample, dtype="float32")
+        )
+
+    def update(
+        self,
+        examples: Iterable[Example],
+        *,
+        drop: float = 0.0,
+        sgd: Optional[Optimizer] = None,
+        losses: Optional[Dict[str, float]] = None,
+    ) -> Dict[str, float]:
+        """Learn from a batch of documents and gold-standard information,
+        updating the pipe's model. Delegates to predict and get_loss.
+
+        examples (Iterable[Example]): A batch of Example objects.
+        drop (float): The dropout rate.
+        sgd (thinc.api.Optimizer): The optimizer.
+        losses (Dict[str, float]): Optional record of the loss during training.
+            Updated using the component name as the key.
+        RETURNS (Dict[str, float]): The updated losses dictionary.
+
+        DOCS: https://spacy.io/api/entitylinker#update
+        """
+        self.validate_kb()
+        if losses is None:
+            losses = {}
+        losses.setdefault(self.name, 0.0)
+        if not examples:
+            return losses
+        validate_examples(examples, "EntityLinker.update")
+        sentence_docs = []
+        for eg in examples:
+            sentences = [s for s in eg.reference.sents]
+            kb_ids = eg.get_aligned("ENT_KB_ID", as_string=True)
+            for ent in eg.reference.ents:
+                # KB ID of the first token is the same as the whole span
+                kb_id = kb_ids[ent.start]
+                if kb_id:
+                    try:
+                        # find the sentence in the list of sentences.
+                        sent_index = sentences.index(ent.sent)
+                    except AttributeError:
+                        # Catch the exception when ent.sent is None and provide a user-friendly warning
+                        raise RuntimeError(Errors.E030) from None
+                    # get n previous sentences, if there are any
+                    start_sentence = max(0, sent_index - self.n_sents)
+                    # get n posterior sentences, or as many < n as there are
+                    end_sentence = min(len(sentences) - 1, sent_index + self.n_sents)
+                    # get token positions
+                    start_token = sentences[start_sentence].start
+                    end_token = sentences[end_sentence].end
+                    # append that span as a doc to training
+                    sent_doc = eg.predicted[start_token:end_token].as_doc()
+                    sentence_docs.append(sent_doc)
+        set_dropout_rate(self.model, drop)
+        if not sentence_docs:
+            warnings.warn(Warnings.W093.format(name="Entity Linker"))
+            return losses
+        sentence_encodings, bp_context = self.model.begin_update(sentence_docs)
+        loss, d_scores = self.get_loss(
+            sentence_encodings=sentence_encodings, examples=examples
+        )
+        bp_context(d_scores)
+        if sgd is not None:
+            self.finish_update(sgd)
+        losses[self.name] += loss
+        return losses
+
+    def get_loss(self, examples: Iterable[Example], sentence_encodings: Floats2d):
+        validate_examples(examples, "EntityLinker.get_loss")
+        entity_encodings = []
+        for eg in examples:
+            kb_ids = eg.get_aligned("ENT_KB_ID", as_string=True)
+            for ent in eg.reference.ents:
+                kb_id = kb_ids[ent.start]
+                if kb_id:
+                    entity_encoding = self.kb.get_vector(kb_id)
+                    entity_encodings.append(entity_encoding)
+        entity_encodings = self.model.ops.asarray(entity_encodings, dtype="float32")
+        if sentence_encodings.shape != entity_encodings.shape:
+            err = Errors.E147.format(
+                method="get_loss", msg="gold entities do not match up"
+            )
+            raise RuntimeError(err)
+        # TODO: fix typing issue here
+        gradients = self.distance.get_grad(sentence_encodings, entity_encodings)  # type: ignore
+        loss = self.distance.get_loss(sentence_encodings, entity_encodings)  # type: ignore
+        loss = loss / len(entity_encodings)
+        return float(loss), gradients
+
+    def predict(self, docs: Iterable[Doc]) -> List[str]:
+        """Apply the pipeline's model to a batch of docs, without modifying them.
+        Returns the KB IDs for each entity in each doc, including NIL if there is
+        no prediction.
+
+        docs (Iterable[Doc]): The documents to predict.
+        RETURNS (List[str]): The models prediction for each document.
+
+        DOCS: https://spacy.io/api/entitylinker#predict
+        """
+        self.validate_kb()
+        entity_count = 0
+        final_kb_ids: List[str] = []
+        if not docs:
+            return final_kb_ids
+        if isinstance(docs, Doc):
+            docs = [docs]
+        for i, doc in enumerate(docs):
+            sentences = [s for s in doc.sents]
+            if len(doc) > 0:
+                # Looping through each entity (TODO: rewrite)
+                for ent in doc.ents:
+                    sent = ent.sent
+                    sent_index = sentences.index(sent)
+                    assert sent_index >= 0
+                    # get n_neighbour sentences, clipped to the length of the document
+                    start_sentence = max(0, sent_index - self.n_sents)
+                    end_sentence = min(len(sentences) - 1, sent_index + self.n_sents)
+                    start_token = sentences[start_sentence].start
+                    end_token = sentences[end_sentence].end
+                    sent_doc = doc[start_token:end_token].as_doc()
+                    # currently, the context is the same for each entity in a sentence (should be refined)
+                    xp = self.model.ops.xp
+                    if self.incl_context:
+                        sentence_encoding = self.model.predict([sent_doc])[0]
+                        sentence_encoding_t = sentence_encoding.T
+                        sentence_norm = xp.linalg.norm(sentence_encoding_t)
+                    entity_count += 1
+                    if ent.label_ in self.labels_discard:
+                        # ignoring this entity - setting to NIL
+                        final_kb_ids.append(self.NIL)
+                    else:
+                        candidates = list(self.get_candidates(self.kb, ent))
+                        if not candidates:
+                            # no prediction possible for this entity - setting to NIL
+                            final_kb_ids.append(self.NIL)
+                        elif len(candidates) == 1:
+                            # shortcut for efficiency reasons: take the 1 candidate
+                            # TODO: thresholding
+                            final_kb_ids.append(candidates[0].entity_)
+                        else:
+                            random.shuffle(candidates)
+                            # set all prior probabilities to 0 if incl_prior=False
+                            prior_probs = xp.asarray([c.prior_prob for c in candidates])
+                            if not self.incl_prior:
+                                prior_probs = xp.asarray([0.0 for _ in candidates])
+                            scores = prior_probs
+                            # add in similarity from the context
+                            if self.incl_context:
+                                entity_encodings = xp.asarray(
+                                    [c.entity_vector for c in candidates]
+                                )
+                                entity_norm = xp.linalg.norm(entity_encodings, axis=1)
+                                if len(entity_encodings) != len(prior_probs):
+                                    raise RuntimeError(
+                                        Errors.E147.format(
+                                            method="predict",
+                                            msg="vectors not of equal length",
+                                        )
+                                    )
+                                # cosine similarity
+                                sims = xp.dot(entity_encodings, sentence_encoding_t) / (
+                                    sentence_norm * entity_norm
+                                )
+                                if sims.shape != prior_probs.shape:
+                                    raise ValueError(Errors.E161)
+                                scores = prior_probs + sims - (prior_probs * sims)
+                            # TODO: thresholding
+                            best_index = scores.argmax().item()
+                            best_candidate = candidates[best_index]
+                            final_kb_ids.append(best_candidate.entity_)
+        if not (len(final_kb_ids) == entity_count):
+            err = Errors.E147.format(
+                method="predict", msg="result variables not of equal length"
+            )
+            raise RuntimeError(err)
+        return final_kb_ids
+
+    def set_annotations(self, docs: Iterable[Doc], kb_ids: List[str]) -> None:
+        """Modify a batch of documents, using pre-computed scores.
+
+        docs (Iterable[Doc]): The documents to modify.
+        kb_ids (List[str]): The IDs to set, produced by EntityLinker.predict.
+
+        DOCS: https://spacy.io/api/entitylinker#set_annotations
+        """
+        count_ents = len([ent for doc in docs for ent in doc.ents])
+        if count_ents != len(kb_ids):
+            raise ValueError(Errors.E148.format(ents=count_ents, ids=len(kb_ids)))
+        i = 0
+        overwrite = self.cfg["overwrite"]
+        for doc in docs:
+            for ent in doc.ents:
+                kb_id = kb_ids[i]
+                i += 1
+                for token in ent:
+                    if token.ent_kb_id == 0 or overwrite:
+                        token.ent_kb_id_ = kb_id
+
+    def to_bytes(self, *, exclude=tuple()):
+        """Serialize the pipe to a bytestring.
+
+        exclude (Iterable[str]): String names of serialization fields to exclude.
+        RETURNS (bytes): The serialized object.
+
+        DOCS: https://spacy.io/api/entitylinker#to_bytes
+        """
+        self._validate_serialization_attrs()
+        serialize = {}
+        if hasattr(self, "cfg") and self.cfg is not None:
+            serialize["cfg"] = lambda: srsly.json_dumps(self.cfg)
+        serialize["vocab"] = lambda: self.vocab.to_bytes(exclude=exclude)
+        serialize["kb"] = self.kb.to_bytes
+        serialize["model"] = self.model.to_bytes
+        return util.to_bytes(serialize, exclude)
+
+    def from_bytes(self, bytes_data, *, exclude=tuple()):
+        """Load the pipe from a bytestring.
+
+        exclude (Iterable[str]): String names of serialization fields to exclude.
+        RETURNS (TrainablePipe): The loaded object.
+
+        DOCS: https://spacy.io/api/entitylinker#from_bytes
+        """
+        self._validate_serialization_attrs()
+
+        def load_model(b):
+            try:
+                self.model.from_bytes(b)
+            except AttributeError:
+                raise ValueError(Errors.E149) from None
+
+        deserialize = {}
+        if hasattr(self, "cfg") and self.cfg is not None:
+            deserialize["cfg"] = lambda b: self.cfg.update(srsly.json_loads(b))
+        deserialize["vocab"] = lambda b: self.vocab.from_bytes(b, exclude=exclude)
+        deserialize["kb"] = lambda b: self.kb.from_bytes(b)
+        deserialize["model"] = load_model
+        util.from_bytes(bytes_data, deserialize, exclude)
+        return self
+
+    def to_disk(
+        self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList()
+    ) -> None:
+        """Serialize the pipe to disk.
+
+        path (str / Path): Path to a directory.
+        exclude (Iterable[str]): String names of serialization fields to exclude.
+
+        DOCS: https://spacy.io/api/entitylinker#to_disk
+        """
+        serialize = {}
+        serialize["vocab"] = lambda p: self.vocab.to_disk(p, exclude=exclude)
+        serialize["cfg"] = lambda p: srsly.write_json(p, self.cfg)
+        serialize["kb"] = lambda p: self.kb.to_disk(p)
+        serialize["model"] = lambda p: self.model.to_disk(p)
+        util.to_disk(path, serialize, exclude)
+
+    def from_disk(
+        self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList()
+    ) -> "EntityLinker":
+        """Load the pipe from disk. Modifies the object in place and returns it.
+
+        path (str / Path): Path to a directory.
+        exclude (Iterable[str]): String names of serialization fields to exclude.
+        RETURNS (EntityLinker): The modified EntityLinker object.
+
+        DOCS: https://spacy.io/api/entitylinker#from_disk
+        """
+
+        def load_model(p):
+            try:
+                with p.open("rb") as infile:
+                    self.model.from_bytes(infile.read())
+            except AttributeError:
+                raise ValueError(Errors.E149) from None
+
+        deserialize: Dict[str, Callable[[Any], Any]] = {}
+        deserialize["cfg"] = lambda p: self.cfg.update(deserialize_config(p))
+        deserialize["vocab"] = lambda p: self.vocab.from_disk(p, exclude=exclude)
+        deserialize["kb"] = lambda p: self.kb.from_disk(p)
+        deserialize["model"] = load_model
+        util.from_disk(path, deserialize, exclude)
+        return self
+
+    def rehearse(self, examples, *, sgd=None, losses=None, **config):
+        raise NotImplementedError
+
+    def add_label(self, label):
+        raise NotImplementedError

cgpenv/Lib/site-packages/spacy/pipeline/entityruler.py

+import warnings
+from typing import Optional, Union, List, Dict, Tuple, Iterable, Any, Callable, Sequence
+from typing import cast
+from collections import defaultdict
+from pathlib import Path
+import srsly
+
+from .pipe import Pipe
+from ..training import Example
+from ..language import Language
+from ..errors import Errors, Warnings
+from ..util import ensure_path, to_disk, from_disk, SimpleFrozenList, registry
+from ..tokens import Doc, Span
+from ..matcher import Matcher, PhraseMatcher
+from ..scorer import get_ner_prf
+
+
+DEFAULT_ENT_ID_SEP = "||"
+PatternType = Dict[str, Union[str, List[Dict[str, Any]]]]
+
+
+@Language.factory(
+    "entity_ruler",
+    assigns=["doc.ents", "token.ent_type", "token.ent_iob"],
+    default_config={
+        "phrase_matcher_attr": None,
+        "validate": False,
+        "overwrite_ents": False,
+        "ent_id_sep": DEFAULT_ENT_ID_SEP,
+        "scorer": {"@scorers": "spacy.entity_ruler_scorer.v1"},
+    },
+    default_score_weights={
+        "ents_f": 1.0,
+        "ents_p": 0.0,
+        "ents_r": 0.0,
+        "ents_per_type": None,
+    },
+)
+def make_entity_ruler(
+    nlp: Language,
+    name: str,
+    phrase_matcher_attr: Optional[Union[int, str]],
+    validate: bool,
+    overwrite_ents: bool,
+    ent_id_sep: str,
+    scorer: Optional[Callable],
+):
+    return EntityRuler(
+        nlp,
+        name,
+        phrase_matcher_attr=phrase_matcher_attr,
+        validate=validate,
+        overwrite_ents=overwrite_ents,
+        ent_id_sep=ent_id_sep,
+        scorer=scorer,
+    )
+
+
+def entity_ruler_score(examples, **kwargs):
+    return get_ner_prf(examples)
+
+
+@registry.scorers("spacy.entity_ruler_scorer.v1")
+def make_entity_ruler_scorer():
+    return entity_ruler_score
+
+
+class EntityRuler(Pipe):
+    """The EntityRuler lets you add spans to the `Doc.ents` using token-based
+    rules or exact phrase matches. It can be combined with the statistical
+    `EntityRecognizer` to boost accuracy, or used on its own to implement a
+    purely rule-based entity recognition system. After initialization, the
+    component is typically added to the pipeline using `nlp.add_pipe`.
+
+    DOCS: https://spacy.io/api/entityruler
+    USAGE: https://spacy.io/usage/rule-based-matching#entityruler
+    """
+
+    def __init__(
+        self,
+        nlp: Language,
+        name: str = "entity_ruler",
+        *,
+        phrase_matcher_attr: Optional[Union[int, str]] = None,
+        validate: bool = False,
+        overwrite_ents: bool = False,
+        ent_id_sep: str = DEFAULT_ENT_ID_SEP,
+        patterns: Optional[List[PatternType]] = None,
+        scorer: Optional[Callable] = entity_ruler_score,
+    ) -> None:
+        """Initialize the entity ruler. If patterns are supplied here, they
+        need to be a list of dictionaries with a `"label"` and `"pattern"`
+        key. A pattern can either be a token pattern (list) or a phrase pattern
+        (string). For example: `{'label': 'ORG', 'pattern': 'Apple'}`.
+
+        nlp (Language): The shared nlp object to pass the vocab to the matchers
+            and process phrase patterns.
+        name (str): Instance name of the current pipeline component. Typically
+            passed in automatically from the factory when the component is
+            added. Used to disable the current entity ruler while creating
+            phrase patterns with the nlp object.
+        phrase_matcher_attr (int / str): Token attribute to match on, passed
+            to the internal PhraseMatcher as `attr`
+        validate (bool): Whether patterns should be validated, passed to
+            Matcher and PhraseMatcher as `validate`
+        patterns (iterable): Optional patterns to load in.
+        overwrite_ents (bool): If existing entities are present, e.g. entities
+            added by the model, overwrite them by matches if necessary.
+        ent_id_sep (str): Separator used internally for entity IDs.
+        scorer (Optional[Callable]): The scoring method. Defaults to
+            spacy.scorer.get_ner_prf.
+
+        DOCS: https://spacy.io/api/entityruler#init
+        """
+        self.nlp = nlp
+        self.name = name
+        self.overwrite = overwrite_ents
+        self.token_patterns = defaultdict(list)  # type: ignore
+        self.phrase_patterns = defaultdict(list)  # type: ignore
+        self._validate = validate
+        self.matcher = Matcher(nlp.vocab, validate=validate)
+        self.phrase_matcher_attr = phrase_matcher_attr
+        self.phrase_matcher = PhraseMatcher(
+            nlp.vocab, attr=self.phrase_matcher_attr, validate=validate
+        )
+        self.ent_id_sep = ent_id_sep
+        self._ent_ids = defaultdict(tuple)  # type: ignore
+        if patterns is not None:
+            self.add_patterns(patterns)
+        self.scorer = scorer
+
+    def __len__(self) -> int:
+        """The number of all patterns added to the entity ruler."""
+        n_token_patterns = sum(len(p) for p in self.token_patterns.values())
+        n_phrase_patterns = sum(len(p) for p in self.phrase_patterns.values())
+        return n_token_patterns + n_phrase_patterns
+
+    def __contains__(self, label: str) -> bool:
+        """Whether a label is present in the patterns."""
+        return label in self.token_patterns or label in self.phrase_patterns
+
+    def __call__(self, doc: Doc) -> Doc:
+        """Find matches in document and add them as entities.
+
+        doc (Doc): The Doc object in the pipeline.
+        RETURNS (Doc): The Doc with added entities, if available.
+
+        DOCS: https://spacy.io/api/entityruler#call
+        """
+        error_handler = self.get_error_handler()
+        try:
+            matches = self.match(doc)
+            self.set_annotations(doc, matches)
+            return doc
+        except Exception as e:
+            return error_handler(self.name, self, [doc], e)
+
+    def match(self, doc: Doc):
+        self._require_patterns()
+        with warnings.catch_warnings():
+            warnings.filterwarnings("ignore", message="\\[W036")
+            matches = cast(
+                List[Tuple[int, int, int]],
+                list(self.matcher(doc)) + list(self.phrase_matcher(doc)),
+            )
+        final_matches = set(
+            [(m_id, start, end) for m_id, start, end in matches if start != end]
+        )
+        get_sort_key = lambda m: (m[2] - m[1], -m[1])
+        final_matches = sorted(final_matches, key=get_sort_key, reverse=True)
+        return final_matches
+
+    def set_annotations(self, doc, matches):
+        """Modify the document in place"""
+        entities = list(doc.ents)
+        new_entities = []
+        seen_tokens = set()
+        for match_id, start, end in matches:
+            if any(t.ent_type for t in doc[start:end]) and not self.overwrite:
+                continue
+            # check for end - 1 here because boundaries are inclusive
+            if start not in seen_tokens and end - 1 not in seen_tokens:
+                if match_id in self._ent_ids:
+                    label, ent_id = self._ent_ids[match_id]
+                    span = Span(doc, start, end, label=label)
+                    if ent_id:
+                        for token in span:
+                            token.ent_id_ = ent_id
+                else:
+                    span = Span(doc, start, end, label=match_id)
+                new_entities.append(span)
+                entities = [
+                    e for e in entities if not (e.start < end and e.end > start)
+                ]
+                seen_tokens.update(range(start, end))
+        doc.ents = entities + new_entities
+
+    @property
+    def labels(self) -> Tuple[str, ...]:
+        """All labels present in the match patterns.
+
+        RETURNS (set): The string labels.
+
+        DOCS: https://spacy.io/api/entityruler#labels
+        """
+        keys = set(self.token_patterns.keys())
+        keys.update(self.phrase_patterns.keys())
+        all_labels = set()
+
+        for l in keys:
+            if self.ent_id_sep in l:
+                label, _ = self._split_label(l)
+                all_labels.add(label)
+            else:
+                all_labels.add(l)
+        return tuple(sorted(all_labels))
+
+    def initialize(
+        self,
+        get_examples: Callable[[], Iterable[Example]],
+        *,
+        nlp: Optional[Language] = None,
+        patterns: Optional[Sequence[PatternType]] = None,
+    ):
+        """Initialize the pipe for training.
+
+        get_examples (Callable[[], Iterable[Example]]): Function that
+            returns a representative sample of gold-standard Example objects.
+        nlp (Language): The current nlp object the component is part of.
+        patterns Optional[Iterable[PatternType]]: The list of patterns.
+
+        DOCS: https://spacy.io/api/entityruler#initialize
+        """
+        self.clear()
+        if patterns:
+            self.add_patterns(patterns)  # type: ignore[arg-type]
+
+    @property
+    def ent_ids(self) -> Tuple[Optional[str], ...]:
+        """All entity ids present in the match patterns `id` properties
+
+        RETURNS (set): The string entity ids.
+
+        DOCS: https://spacy.io/api/entityruler#ent_ids
+        """
+        keys = set(self.token_patterns.keys())
+        keys.update(self.phrase_patterns.keys())
+        all_ent_ids = set()
+
+        for l in keys:
+            if self.ent_id_sep in l:
+                _, ent_id = self._split_label(l)
+                all_ent_ids.add(ent_id)
+        return tuple(all_ent_ids)
+
+    @property
+    def patterns(self) -> List[PatternType]:
+        """Get all patterns that were added to the entity ruler.
+
+        RETURNS (list): The original patterns, one dictionary per pattern.
+
+        DOCS: https://spacy.io/api/entityruler#patterns
+        """
+        all_patterns = []
+        for label, patterns in self.token_patterns.items():
+            for pattern in patterns:
+                ent_label, ent_id = self._split_label(label)
+                p = {"label": ent_label, "pattern": pattern}
+                if ent_id:
+                    p["id"] = ent_id
+                all_patterns.append(p)
+        for label, patterns in self.phrase_patterns.items():
+            for pattern in patterns:
+                ent_label, ent_id = self._split_label(label)
+                p = {"label": ent_label, "pattern": pattern.text}
+                if ent_id:
+                    p["id"] = ent_id
+                all_patterns.append(p)
+        return all_patterns
+
+    def add_patterns(self, patterns: List[PatternType]) -> None:
+        """Add patterns to the entity ruler. A pattern can either be a token
+        pattern (list of dicts) or a phrase pattern (string). For example:
+        {'label': 'ORG', 'pattern': 'Apple'}
+        {'label': 'GPE', 'pattern': [{'lower': 'san'}, {'lower': 'francisco'}]}
+
+        patterns (list): The patterns to add.
+
+        DOCS: https://spacy.io/api/entityruler#add_patterns
+        """
+
+        # disable the nlp components after this one in case they hadn't been initialized / deserialised yet
+        try:
+            current_index = -1
+            for i, (name, pipe) in enumerate(self.nlp.pipeline):
+                if self == pipe:
+                    current_index = i
+                    break
+            subsequent_pipes = [pipe for pipe in self.nlp.pipe_names[current_index:]]
+        except ValueError:
+            subsequent_pipes = []
+        with self.nlp.select_pipes(disable=subsequent_pipes):
+            token_patterns = []
+            phrase_pattern_labels = []
+            phrase_pattern_texts = []
+            phrase_pattern_ids = []
+            for entry in patterns:
+                if isinstance(entry["pattern"], str):
+                    phrase_pattern_labels.append(entry["label"])
+                    phrase_pattern_texts.append(entry["pattern"])
+                    phrase_pattern_ids.append(entry.get("id"))
+                elif isinstance(entry["pattern"], list):
+                    token_patterns.append(entry)
+            phrase_patterns = []
+            for label, pattern, ent_id in zip(
+                phrase_pattern_labels,
+                self.nlp.pipe(phrase_pattern_texts),
+                phrase_pattern_ids,
+            ):
+                phrase_pattern = {"label": label, "pattern": pattern}
+                if ent_id:
+                    phrase_pattern["id"] = ent_id
+                phrase_patterns.append(phrase_pattern)
+            for entry in token_patterns + phrase_patterns:  # type: ignore[operator]
+                label = entry["label"]
+                if "id" in entry:
+                    ent_label = label
+                    label = self._create_label(label, entry["id"])
+                    key = self.matcher._normalize_key(label)
+                    self._ent_ids[key] = (ent_label, entry["id"])
+                pattern = entry["pattern"]  # type: ignore
+                if isinstance(pattern, Doc):
+                    self.phrase_patterns[label].append(pattern)
+                    self.phrase_matcher.add(label, [pattern])  # type: ignore
+                elif isinstance(pattern, list):
+                    self.token_patterns[label].append(pattern)
+                    self.matcher.add(label, [pattern])
+                else:
+                    raise ValueError(Errors.E097.format(pattern=pattern))
+
+    def clear(self) -> None:
+        """Reset all patterns."""
+        self.token_patterns = defaultdict(list)
+        self.phrase_patterns = defaultdict(list)
+        self._ent_ids = defaultdict(tuple)
+        self.matcher = Matcher(self.nlp.vocab, validate=self._validate)
+        self.phrase_matcher = PhraseMatcher(
+            self.nlp.vocab, attr=self.phrase_matcher_attr, validate=self._validate
+        )
+
+    def _require_patterns(self) -> None:
+        """Raise a warning if this component has no patterns defined."""
+        if len(self) == 0:
+            warnings.warn(Warnings.W036.format(name=self.name))
+
+    def _split_label(self, label: str) -> Tuple[str, Optional[str]]:
+        """Split Entity label into ent_label and ent_id if it contains self.ent_id_sep
+
+        label (str): The value of label in a pattern entry
+        RETURNS (tuple): ent_label, ent_id
+        """
+        if self.ent_id_sep in label:
+            ent_label, ent_id = label.rsplit(self.ent_id_sep, 1)
+        else:
+            ent_label = label
+            ent_id = None  # type: ignore
+        return ent_label, ent_id
+
+    def _create_label(self, label: Any, ent_id: Any) -> str:
+        """Join Entity label with ent_id if the pattern has an `id` attribute
+        If ent_id is not a string, the label is returned as is.
+
+        label (str): The label to set for ent.label_
+        ent_id (str): The label
+        RETURNS (str): The ent_label joined with configured `ent_id_sep`
+        """
+        if isinstance(ent_id, str):
+            label = f"{label}{self.ent_id_sep}{ent_id}"
+        return label
+
+    def from_bytes(
+        self, patterns_bytes: bytes, *, exclude: Iterable[str] = SimpleFrozenList()
+    ) -> "EntityRuler":
+        """Load the entity ruler from a bytestring.
+
+        patterns_bytes (bytes): The bytestring to load.
+        RETURNS (EntityRuler): The loaded entity ruler.
+
+        DOCS: https://spacy.io/api/entityruler#from_bytes
+        """
+        cfg = srsly.msgpack_loads(patterns_bytes)
+        self.clear()
+        if isinstance(cfg, dict):
+            self.add_patterns(cfg.get("patterns", cfg))
+            self.overwrite = cfg.get("overwrite", False)
+            self.phrase_matcher_attr = cfg.get("phrase_matcher_attr", None)
+            self.phrase_matcher = PhraseMatcher(
+                self.nlp.vocab, attr=self.phrase_matcher_attr
+            )
+            self.ent_id_sep = cfg.get("ent_id_sep", DEFAULT_ENT_ID_SEP)
+        else:
+            self.add_patterns(cfg)
+        return self
+
+    def to_bytes(self, *, exclude: Iterable[str] = SimpleFrozenList()) -> bytes:
+        """Serialize the entity ruler patterns to a bytestring.
+
+        RETURNS (bytes): The serialized patterns.
+
+        DOCS: https://spacy.io/api/entityruler#to_bytes
+        """
+        serial = {
+            "overwrite": self.overwrite,
+            "ent_id_sep": self.ent_id_sep,
+            "phrase_matcher_attr": self.phrase_matcher_attr,
+            "patterns": self.patterns,
+        }
+        return srsly.msgpack_dumps(serial)
+
+    def from_disk(
+        self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList()
+    ) -> "EntityRuler":
+        """Load the entity ruler from a file. Expects a file containing
+        newline-delimited JSON (JSONL) with one entry per line.
+
+        path (str / Path): The JSONL file to load.
+        RETURNS (EntityRuler): The loaded entity ruler.
+
+        DOCS: https://spacy.io/api/entityruler#from_disk
+        """
+        path = ensure_path(path)
+        self.clear()
+        depr_patterns_path = path.with_suffix(".jsonl")
+        if depr_patterns_path.is_file():
+            patterns = srsly.read_jsonl(depr_patterns_path)
+            self.add_patterns(patterns)
+        else:
+            cfg = {}
+            deserializers_patterns = {
+                "patterns": lambda p: self.add_patterns(
+                    srsly.read_jsonl(p.with_suffix(".jsonl"))
+                )
+            }
+            deserializers_cfg = {"cfg": lambda p: cfg.update(srsly.read_json(p))}
+            from_disk(path, deserializers_cfg, {})
+            self.overwrite = cfg.get("overwrite", False)
+            self.phrase_matcher_attr = cfg.get("phrase_matcher_attr")
+            self.ent_id_sep = cfg.get("ent_id_sep", DEFAULT_ENT_ID_SEP)
+
+            self.phrase_matcher = PhraseMatcher(
+                self.nlp.vocab, attr=self.phrase_matcher_attr
+            )
+            from_disk(path, deserializers_patterns, {})
+        return self
+
+    def to_disk(
+        self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList()
+    ) -> None:
+        """Save the entity ruler patterns to a directory. The patterns will be
+        saved as newline-delimited JSON (JSONL).
+
+        path (str / Path): The JSONL file to save.
+
+        DOCS: https://spacy.io/api/entityruler#to_disk
+        """
+        path = ensure_path(path)
+        cfg = {
+            "overwrite": self.overwrite,
+            "phrase_matcher_attr": self.phrase_matcher_attr,
+            "ent_id_sep": self.ent_id_sep,
+        }
+        serializers = {
+            "patterns": lambda p: srsly.write_jsonl(
+                p.with_suffix(".jsonl"), self.patterns
+            ),
+            "cfg": lambda p: srsly.write_json(p, cfg),
+        }
+        if path.suffix == ".jsonl":  # user wants to save only JSONL
+            srsly.write_jsonl(path, self.patterns)
+        else:
+            to_disk(path, serializers, {})

cgpenv/Lib/site-packages/spacy/pipeline/functions.py

+from typing import Dict, Any
+import srsly
+
+from ..language import Language
+from ..matcher import Matcher
+from ..tokens import Doc
+from .. import util
+
+
+@Language.component(
+    "merge_noun_chunks",
+    requires=["token.dep", "token.tag", "token.pos"],
+    retokenizes=True,
+)
+def merge_noun_chunks(doc: Doc) -> Doc:
+    """Merge noun chunks into a single token.
+
+    doc (Doc): The Doc object.
+    RETURNS (Doc): The Doc object with merged noun chunks.
+
+    DOCS: https://spacy.io/api/pipeline-functions#merge_noun_chunks
+    """
+    if not doc.has_annotation("DEP"):
+        return doc
+    with doc.retokenize() as retokenizer:
+        for np in doc.noun_chunks:
+            attrs = {"tag": np.root.tag, "dep": np.root.dep}
+            retokenizer.merge(np, attrs=attrs)  # type: ignore[arg-type]
+    return doc
+
+
+@Language.component(
+    "merge_entities",
+    requires=["doc.ents", "token.ent_iob", "token.ent_type"],
+    retokenizes=True,
+)
+def merge_entities(doc: Doc):
+    """Merge entities into a single token.
+
+    doc (Doc): The Doc object.
+    RETURNS (Doc): The Doc object with merged entities.
+
+    DOCS: https://spacy.io/api/pipeline-functions#merge_entities
+    """
+    with doc.retokenize() as retokenizer:
+        for ent in doc.ents:
+            attrs = {"tag": ent.root.tag, "dep": ent.root.dep, "ent_type": ent.label}
+            retokenizer.merge(ent, attrs=attrs)  # type: ignore[arg-type]
+    return doc
+
+
+@Language.component("merge_subtokens", requires=["token.dep"], retokenizes=True)
+def merge_subtokens(doc: Doc, label: str = "subtok") -> Doc:
+    """Merge subtokens into a single token.
+
+    doc (Doc): The Doc object.
+    label (str): The subtoken dependency label.
+    RETURNS (Doc): The Doc object with merged subtokens.
+
+    DOCS: https://spacy.io/api/pipeline-functions#merge_subtokens
+    """
+    # TODO: make stateful component with "label" config
+    merger = Matcher(doc.vocab)
+    merger.add("SUBTOK", [[{"DEP": label, "op": "+"}]])
+    matches = merger(doc)
+    spans = util.filter_spans([doc[start : end + 1] for _, start, end in matches])  # type: ignore[misc, operator]
+    with doc.retokenize() as retokenizer:
+        for span in spans:
+            retokenizer.merge(span)
+    return doc
+
+
+@Language.factory(
+    "token_splitter",
+    default_config={"min_length": 25, "split_length": 10},
+    retokenizes=True,
+)
+def make_token_splitter(
+    nlp: Language, name: str, *, min_length: int = 0, split_length: int = 0
+):
+    return TokenSplitter(min_length=min_length, split_length=split_length)
+
+
+class TokenSplitter:
+    def __init__(self, min_length: int = 0, split_length: int = 0):
+        self.min_length = min_length
+        self.split_length = split_length
+
+    def __call__(self, doc: Doc) -> Doc:
+        if self.min_length > 0 and self.split_length > 0:
+            with doc.retokenize() as retokenizer:
+                for t in doc:
+                    if len(t.text) >= self.min_length:
+                        orths = []
+                        heads = []
+                        attrs = {}  # type: ignore[var-annotated]
+                        for i in range(0, len(t.text), self.split_length):
+                            orths.append(t.text[i : i + self.split_length])
+                            heads.append((t, i / self.split_length))
+                        retokenizer.split(t, orths, heads, attrs)  # type: ignore[arg-type]
+        return doc
+
+    def _get_config(self) -> Dict[str, Any]:
+        return {
+            "min_length": self.min_length,
+            "split_length": self.split_length,
+        }
+
+    def _set_config(self, config: Dict[str, Any] = {}) -> None:
+        self.min_length = config.get("min_length", 0)
+        self.split_length = config.get("split_length", 0)
+
+    def to_bytes(self, **kwargs):
+        serializers = {
+            "cfg": lambda: srsly.json_dumps(self._get_config()),
+        }
+        return util.to_bytes(serializers, [])
+
+    def from_bytes(self, data, **kwargs):
+        deserializers = {
+            "cfg": lambda b: self._set_config(srsly.json_loads(b)),
+        }
+        util.from_bytes(data, deserializers, [])
+        return self
+
+    def to_disk(self, path, **kwargs):
+        path = util.ensure_path(path)
+        serializers = {
+            "cfg": lambda p: srsly.write_json(p, self._get_config()),
+        }
+        return util.to_disk(path, serializers, [])
+
+    def from_disk(self, path, **kwargs):
+        path = util.ensure_path(path)
+        serializers = {
+            "cfg": lambda p: self._set_config(srsly.read_json(p)),
+        }
+        util.from_disk(path, serializers, [])

cgpenv/Lib/site-packages/spacy/pipeline/lemmatizer.py

+from typing import Optional, List, Dict, Any, Callable, Iterable, Union, Tuple
+from thinc.api import Model
+from pathlib import Path
+
+import warnings
+
+from .pipe import Pipe
+from ..errors import Errors, Warnings
+from ..language import Language
+from ..training import Example
+from ..lookups import Lookups, load_lookups
+from ..scorer import Scorer
+from ..tokens import Doc, Token
+from ..vocab import Vocab
+from ..util import logger, SimpleFrozenList, registry
+from .. import util
+
+
+@Language.factory(
+    "lemmatizer",
+    assigns=["token.lemma"],
+    default_config={
+        "model": None,
+        "mode": "lookup",
+        "overwrite": False,
+        "scorer": {"@scorers": "spacy.lemmatizer_scorer.v1"},
+    },
+    default_score_weights={"lemma_acc": 1.0},
+)
+def make_lemmatizer(
+    nlp: Language,
+    model: Optional[Model],
+    name: str,
+    mode: str,
+    overwrite: bool,
+    scorer: Optional[Callable],
+):
+    return Lemmatizer(
+        nlp.vocab, model, name, mode=mode, overwrite=overwrite, scorer=scorer
+    )
+
+
+def lemmatizer_score(examples: Iterable[Example], **kwargs) -> Dict[str, Any]:
+    return Scorer.score_token_attr(examples, "lemma", **kwargs)
+
+
+@registry.scorers("spacy.lemmatizer_scorer.v1")
+def make_lemmatizer_scorer():
+    return lemmatizer_score
+
+
+class Lemmatizer(Pipe):
+    """
+    The Lemmatizer supports simple part-of-speech-sensitive suffix rules and
+    lookup tables.
+
+    DOCS: https://spacy.io/api/lemmatizer
+    """
+
+    @classmethod
+    def get_lookups_config(cls, mode: str) -> Tuple[List[str], List[str]]:
+        """Returns the lookups configuration settings for a given mode for use
+        in Lemmatizer.load_lookups.
+
+        mode (str): The lemmatizer mode.
+        RETURNS (Tuple[List[str], List[str]]): The required and optional
+            lookup tables for this mode.
+        """
+        if mode == "lookup":
+            return (["lemma_lookup"], [])
+        elif mode == "rule":
+            return (["lemma_rules"], ["lemma_exc", "lemma_index"])
+        return ([], [])
+
+    def __init__(
+        self,
+        vocab: Vocab,
+        model: Optional[Model],
+        name: str = "lemmatizer",
+        *,
+        mode: str = "lookup",
+        overwrite: bool = False,
+        scorer: Optional[Callable] = lemmatizer_score,
+    ) -> None:
+        """Initialize a Lemmatizer.
+
+        vocab (Vocab): The vocab.
+        model (Model): A model (not yet implemented).
+        name (str): The component name. Defaults to "lemmatizer".
+        mode (str): The lemmatizer mode: "lookup", "rule". Defaults to "lookup".
+        overwrite (bool): Whether to overwrite existing lemmas. Defaults to
+            `False`.
+        scorer (Optional[Callable]): The scoring method. Defaults to
+            Scorer.score_token_attr for the attribute "lemma".
+
+        DOCS: https://spacy.io/api/lemmatizer#init
+        """
+        self.vocab = vocab
+        self.model = model
+        self.name = name
+        self._mode = mode
+        self.lookups = Lookups()
+        self.overwrite = overwrite
+        self._validated = False
+        if self.mode == "lookup":
+            self.lemmatize = self.lookup_lemmatize
+        elif self.mode == "rule":
+            self.lemmatize = self.rule_lemmatize
+        else:
+            mode_attr = f"{self.mode}_lemmatize"
+            if not hasattr(self, mode_attr):
+                raise ValueError(Errors.E1003.format(mode=mode))
+            self.lemmatize = getattr(self, mode_attr)
+        self.cache = {}  # type: ignore[var-annotated]
+        self.scorer = scorer
+
+    @property
+    def mode(self):
+        return self._mode
+
+    def __call__(self, doc: Doc) -> Doc:
+        """Apply the lemmatizer to one document.
+
+        doc (Doc): The Doc to process.
+        RETURNS (Doc): The processed Doc.
+
+        DOCS: https://spacy.io/api/lemmatizer#call
+        """
+        if not self._validated:
+            self._validate_tables(Errors.E1004)
+        error_handler = self.get_error_handler()
+        try:
+            for token in doc:
+                if self.overwrite or token.lemma == 0:
+                    token.lemma_ = self.lemmatize(token)[0]
+            return doc
+        except Exception as e:
+            error_handler(self.name, self, [doc], e)
+
+    def initialize(
+        self,
+        get_examples: Optional[Callable[[], Iterable[Example]]] = None,
+        *,
+        nlp: Optional[Language] = None,
+        lookups: Optional[Lookups] = None,
+    ):
+        """Initialize the lemmatizer and load in data.
+
+        get_examples (Callable[[], Iterable[Example]]): Function that
+            returns a representative sample of gold-standard Example objects.
+        nlp (Language): The current nlp object the component is part of.
+        lookups (Lookups): The lookups object containing the (optional) tables
+            such as "lemma_rules", "lemma_index", "lemma_exc" and
+            "lemma_lookup". Defaults to None.
+        """
+        required_tables, optional_tables = self.get_lookups_config(self.mode)
+        if lookups is None:
+            logger.debug("Lemmatizer: loading tables from spacy-lookups-data")
+            lookups = load_lookups(lang=self.vocab.lang, tables=required_tables)
+            optional_lookups = load_lookups(
+                lang=self.vocab.lang, tables=optional_tables, strict=False
+            )
+            for table in optional_lookups.tables:
+                lookups.set_table(table, optional_lookups.get_table(table))
+        self.lookups = lookups
+        self._validate_tables(Errors.E1004)
+
+    def _validate_tables(self, error_message: str = Errors.E912) -> None:
+        """Check that the lookups are correct for the current mode."""
+        required_tables, optional_tables = self.get_lookups_config(self.mode)
+        for table in required_tables:
+            if table not in self.lookups:
+                raise ValueError(
+                    error_message.format(
+                        mode=self.mode,
+                        tables=required_tables,
+                        found=self.lookups.tables,
+                    )
+                )
+        self._validated = True
+
+    def lookup_lemmatize(self, token: Token) -> List[str]:
+        """Lemmatize using a lookup-based approach.
+
+        token (Token): The token to lemmatize.
+        RETURNS (list): The available lemmas for the string.
+
+        DOCS: https://spacy.io/api/lemmatizer#lookup_lemmatize
+        """
+        lookup_table = self.lookups.get_table("lemma_lookup", {})
+        result = lookup_table.get(token.text, token.text)
+        if isinstance(result, str):
+            result = [result]
+        return result
+
+    def rule_lemmatize(self, token: Token) -> List[str]:
+        """Lemmatize using a rule-based approach.
+
+        token (Token): The token to lemmatize.
+        RETURNS (list): The available lemmas for the string.
+
+        DOCS: https://spacy.io/api/lemmatizer#rule_lemmatize
+        """
+        cache_key = (token.orth, token.pos, token.morph.key)  # type: ignore[attr-defined]
+        if cache_key in self.cache:
+            return self.cache[cache_key]
+        string = token.text
+        univ_pos = token.pos_.lower()
+        if univ_pos in ("", "eol", "space"):
+            if univ_pos == "":
+                warnings.warn(Warnings.W108)
+            return [string.lower()]
+        # See Issue #435 for example of where this logic is requied.
+        if self.is_base_form(token):
+            return [string.lower()]
+        index_table = self.lookups.get_table("lemma_index", {})
+        exc_table = self.lookups.get_table("lemma_exc", {})
+        rules_table = self.lookups.get_table("lemma_rules", {})
+        if not any(
+            (
+                index_table.get(univ_pos),
+                exc_table.get(univ_pos),
+                rules_table.get(univ_pos),
+            )
+        ):
+            if univ_pos == "propn":
+                return [string]
+            else:
+                return [string.lower()]
+
+        index = index_table.get(univ_pos, {})
+        exceptions = exc_table.get(univ_pos, {})
+        rules = rules_table.get(univ_pos, {})
+        orig = string
+        string = string.lower()
+        forms = []
+        oov_forms = []
+        for old, new in rules:
+            if string.endswith(old):
+                form = string[: len(string) - len(old)] + new
+                if not form:
+                    pass
+                elif form in index or not form.isalpha():
+                    forms.append(form)
+                else:
+                    oov_forms.append(form)
+        # Remove duplicates but preserve the ordering of applied "rules"
+        forms = list(dict.fromkeys(forms))
+        # Put exceptions at the front of the list, so they get priority.
+        # This is a dodgy heuristic -- but it's the best we can do until we get
+        # frequencies on this. We can at least prune out problematic exceptions,
+        # if they shadow more frequent analyses.
+        for form in exceptions.get(string, []):
+            if form not in forms:
+                forms.insert(0, form)
+        if not forms:
+            forms.extend(oov_forms)
+        if not forms:
+            forms.append(orig)
+        self.cache[cache_key] = forms
+        return forms
+
+    def is_base_form(self, token: Token) -> bool:
+        """Check whether the token is a base form that does not need further
+        analysis for lemmatization.
+
+        token (Token): The token.
+        RETURNS (bool): Whether the token is a base form.
+
+        DOCS: https://spacy.io/api/lemmatizer#is_base_form
+        """
+        return False
+
+    def to_disk(
+        self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList()
+    ):
+        """Serialize the pipe to disk.
+
+        path (str / Path): Path to a directory.
+        exclude (Iterable[str]): String names of serialization fields to exclude.
+
+        DOCS: https://spacy.io/api/lemmatizer#to_disk
+        """
+        serialize = {}
+        serialize["vocab"] = lambda p: self.vocab.to_disk(p, exclude=exclude)
+        serialize["lookups"] = lambda p: self.lookups.to_disk(p)
+        util.to_disk(path, serialize, exclude)
+
+    def from_disk(
+        self, path: Union[str, Path], *, exclude: Iterable[str] = SimpleFrozenList()
+    ) -> "Lemmatizer":
+        """Load the pipe from disk. Modifies the object in place and returns it.
+
+        path (str / Path): Path to a directory.
+        exclude (Iterable[str]): String names of serialization fields to exclude.
+        RETURNS (Lemmatizer): The modified Lemmatizer object.
+
+        DOCS: https://spacy.io/api/lemmatizer#from_disk
+        """
+        deserialize: Dict[str, Callable[[Any], Any]] = {}
+        deserialize["vocab"] = lambda p: self.vocab.from_disk(p, exclude=exclude)
+        deserialize["lookups"] = lambda p: self.lookups.from_disk(p)
+        util.from_disk(path, deserialize, exclude)
+        self._validate_tables()
+        return self
+
+    def to_bytes(self, *, exclude: Iterable[str] = SimpleFrozenList()) -> bytes:
+        """Serialize the pipe to a bytestring.
+
+        exclude (Iterable[str]): String names of serialization fields to exclude.
+        RETURNS (bytes): The serialized object.
+
+        DOCS: https://spacy.io/api/lemmatizer#to_bytes
+        """
+        serialize = {}
+        serialize["vocab"] = lambda: self.vocab.to_bytes(exclude=exclude)
+        serialize["lookups"] = self.lookups.to_bytes
+        return util.to_bytes(serialize, exclude)
+
+    def from_bytes(
+        self, bytes_data: bytes, *, exclude: Iterable[str] = SimpleFrozenList()
+    ) -> "Lemmatizer":
+        """Load the pipe from a bytestring.
+
+        bytes_data (bytes): The serialized pipe.
+        exclude (Iterable[str]): String names of serialization fields to exclude.
+        RETURNS (Lemmatizer): The loaded Lemmatizer.
+
+        DOCS: https://spacy.io/api/lemmatizer#from_bytes
+        """
+        deserialize: Dict[str, Callable[[Any], Any]] = {}
+        deserialize["vocab"] = lambda b: self.vocab.from_bytes(b, exclude=exclude)
+        deserialize["lookups"] = lambda b: self.lookups.from_bytes(b)
+        util.from_bytes(bytes_data, deserialize, exclude)
+        self._validate_tables()
+        return self

cgpenv/Lib/site-packages/spacy/pipeline/morphologizer.pyx

+# cython: infer_types=True, profile=True, binding=True
+from typing import Optional, Union, Dict, Callable
+import srsly
+from thinc.api import SequenceCategoricalCrossentropy, Model, Config
+from itertools import islice
+
+from ..tokens.doc cimport Doc
+from ..vocab cimport Vocab
+from ..morphology cimport Morphology
+
+from ..parts_of_speech import IDS as POS_IDS
+from ..symbols import POS
+from ..language import Language
+from ..errors import Errors
+from .pipe import deserialize_config
+from .tagger import Tagger
+from .. import util
+from ..scorer import Scorer
+from ..training import validate_examples, validate_get_examples
+from ..util import registry
+
+# See #9050
+BACKWARD_OVERWRITE = True
+BACKWARD_EXTEND = False
+
+default_model_config = """
+[model]
+@architectures = "spacy.Tagger.v1"
+
+[model.tok2vec]
+@architectures = "spacy.Tok2Vec.v2"
+
+[model.tok2vec.embed]
+@architectures = "spacy.CharacterEmbed.v2"
+width = 128
+rows = 7000
+nM = 64
+nC = 8
+include_static_vectors = false
+
+[model.tok2vec.encode]
+@architectures = "spacy.MaxoutWindowEncoder.v2"
+width = 128
+depth = 4
+window_size = 1
+maxout_pieces = 3
+"""
+
+DEFAULT_MORPH_MODEL = Config().from_str(default_model_config)["model"]
+
+
+@Language.factory(
+    "morphologizer",
+    assigns=["token.morph", "token.pos"],
+    default_config={"model": DEFAULT_MORPH_MODEL, "overwrite": True, "extend": False, "scorer": {"@scorers": "spacy.morphologizer_scorer.v1"}},
+    default_score_weights={"pos_acc": 0.5, "morph_acc": 0.5, "morph_per_feat": None},
+)
+def make_morphologizer(
+    nlp: Language,
+    model: Model,
+    name: str,
+    overwrite: bool,
+    extend: bool,
+    scorer: Optional[Callable],
+):
+    return Morphologizer(nlp.vocab, model, name, overwrite=overwrite, extend=extend, scorer=scorer)
+
+
+def morphologizer_score(examples, **kwargs):
+    def morph_key_getter(token, attr):
+        return getattr(token, attr).key
+
+    results = {}
+    results.update(Scorer.score_token_attr(examples, "pos", **kwargs))
+    results.update(Scorer.score_token_attr(examples, "morph", getter=morph_key_getter, **kwargs))
+    results.update(Scorer.score_token_attr_per_feat(examples,
+        "morph", getter=morph_key_getter, **kwargs))
+    return results
+
+
+@registry.scorers("spacy.morphologizer_scorer.v1")
+def make_morphologizer_scorer():
+    return morphologizer_score
+
+
+class Morphologizer(Tagger):
+    POS_FEAT = "POS"
+
+    def __init__(
+        self,
+        vocab: Vocab,
+        model: Model,
+        name: str = "morphologizer",
+        *,
+        overwrite: bool = BACKWARD_OVERWRITE,
+        extend: bool = BACKWARD_EXTEND,
+        scorer: Optional[Callable] = morphologizer_score,
+    ):
+        """Initialize a morphologizer.
+
+        vocab (Vocab): The shared vocabulary.
+        model (thinc.api.Model): The Thinc Model powering the pipeline component.
+        name (str): The component instance name, used to add entries to the
+            losses during training.
+        scorer (Optional[Callable]): The scoring method. Defaults to
+            Scorer.score_token_attr for the attributes "pos" and "morph" and
+            Scorer.score_token_attr_per_feat for the attribute "morph".
+
+        DOCS: https://spacy.io/api/morphologizer#init
+        """
+        self.vocab = vocab
+        self.model = model
+        self.name = name
+        self._rehearsal_model = None
+        # to be able to set annotations without string operations on labels,
+        # store mappings from morph+POS labels to token-level annotations:
+        # 1) labels_morph stores a mapping from morph+POS->morph
+        # 2) labels_pos stores a mapping from morph+POS->POS
+        cfg = {
+            "labels_morph": {},
+            "labels_pos": {},
+            "overwrite": overwrite,
+            "extend": extend,
+        }
+        self.cfg = dict(sorted(cfg.items()))
+        self.scorer = scorer
+
+    @property
+    def labels(self):
+        """RETURNS (Tuple[str]): The labels currently added to the component."""
+        return tuple(self.cfg["labels_morph"].keys())
+
+    @property
+    def label_data(self) -> Dict[str, Dict[str, Union[str, float, int, None]]]:
+        """A dictionary with all labels data."""
+        return {"morph": self.cfg["labels_morph"], "pos": self.cfg["labels_pos"]}
+
+    def add_label(self, label):
+        """Add a new label to the pipe.
+
+        label (str): The label to add.
+        RETURNS (int): 0 if label is already present, otherwise 1.
+
+        DOCS: https://spacy.io/api/morphologizer#add_label
+        """
+        if not isinstance(label, str):
+            raise ValueError(Errors.E187)
+        if label in self.labels:
+            return 0
+        self._allow_extra_label()
+        # normalize label
+        norm_label = self.vocab.morphology.normalize_features(label)
+        # extract separate POS and morph tags
+        label_dict = Morphology.feats_to_dict(label)
+        pos = label_dict.get(self.POS_FEAT, "")
+        if self.POS_FEAT in label_dict:
+            label_dict.pop(self.POS_FEAT)
+        # normalize morph string and add to morphology table
+        norm_morph = self.vocab.strings[self.vocab.morphology.add(label_dict)]
+        # add label mappings
+        if norm_label not in self.cfg["labels_morph"]:
+            self.cfg["labels_morph"][norm_label] = norm_morph
+            self.cfg["labels_pos"][norm_label] = POS_IDS[pos]
+        return 1
+
+    def initialize(self, get_examples, *, nlp=None, labels=None):
+        """Initialize the pipe for training, using a representative set
+        of data examples.
+
+        get_examples (Callable[[], Iterable[Example]]): Function that
+            returns a representative sample of gold-standard Example objects.
+        nlp (Language): The current nlp object the component is part of.
+
+        DOCS: https://spacy.io/api/morphologizer#initialize
+        """
+        validate_get_examples(get_examples, "Morphologizer.initialize")
+        util.check_lexeme_norms(self.vocab, "morphologizer")
+        if labels is not None:
+            self.cfg["labels_morph"] = labels["morph"]
+            self.cfg["labels_pos"] = labels["pos"]
+        else:
+            # First, fetch all labels from the data
+            for example in get_examples():
+                for i, token in enumerate(example.reference):
+                    pos = token.pos_
+                    # if both are unset, annotation is missing, so do not add
+                    # an empty label
+                    if pos == "" and not token.has_morph():
+                        continue
+                    morph = str(token.morph)
+                    # create and add the combined morph+POS label
+                    morph_dict = Morphology.feats_to_dict(morph)
+                    if pos:
+                        morph_dict[self.POS_FEAT] = pos
+                    norm_label = self.vocab.strings[self.vocab.morphology.add(morph_dict)]
+                    # add label->morph and label->POS mappings
+                    if norm_label not in self.cfg["labels_morph"]:
+                        self.cfg["labels_morph"][norm_label] = morph
+                        self.cfg["labels_pos"][norm_label] = POS_IDS[pos]
+        if len(self.labels) < 1:
+            raise ValueError(Errors.E143.format(name=self.name))
+        doc_sample = []
+        label_sample = []
+        for example in islice(get_examples(), 10):
+            gold_array = []
+            for i, token in enumerate(example.reference):
+                pos = token.pos_
+                morph = str(token.morph)
+                morph_dict = Morphology.feats_to_dict(morph)
+                if pos:
+                    morph_dict[self.POS_FEAT] = pos
+                norm_label = self.vocab.strings[self.vocab.morphology.add(morph_dict)]
+                gold_array.append([1.0 if label == norm_label else 0.0 for label in self.labels])
+            doc_sample.append(example.x)
+            label_sample.append(self.model.ops.asarray(gold_array, dtype="float32"))
+        assert len(doc_sample) > 0, Errors.E923.format(name=self.name)
+        assert len(label_sample) > 0, Errors.E923.format(name=self.name)
+        self.model.initialize(X=doc_sample, Y=label_sample)
+
+    def set_annotations(self, docs, batch_tag_ids):
+        """Modify a batch of documents, using pre-computed scores.
+
+        docs (Iterable[Doc]): The documents to modify.
+        batch_tag_ids: The IDs to set, produced by Morphologizer.predict.
+
+        DOCS: https://spacy.io/api/morphologizer#set_annotations
+        """
+        if isinstance(docs, Doc):
+            docs = [docs]
+        cdef Doc doc
+        cdef Vocab vocab = self.vocab
+        cdef bint overwrite = self.cfg["overwrite"]
+        cdef bint extend = self.cfg["extend"]
+        for i, doc in enumerate(docs):
+            doc_tag_ids = batch_tag_ids[i]
+            if hasattr(doc_tag_ids, "get"):
+                doc_tag_ids = doc_tag_ids.get()
+            for j, tag_id in enumerate(doc_tag_ids):
+                morph = self.labels[tag_id]
+                # set morph
+                if doc.c[j].morph == 0 or overwrite or extend:
+                    if overwrite and extend:
+                        # morphologizer morph overwrites any existing features
+                        # while extending
+                        extended_morph = Morphology.feats_to_dict(self.vocab.strings[doc.c[j].morph])
+                        extended_morph.update(Morphology.feats_to_dict(self.cfg["labels_morph"].get(morph, 0)))
+                        doc.c[j].morph = self.vocab.morphology.add(extended_morph)
+                    elif extend:
+                        # existing features are preserved and any new features
+                        # are added
+                        extended_morph = Morphology.feats_to_dict(self.cfg["labels_morph"].get(morph, 0))
+                        extended_morph.update(Morphology.feats_to_dict(self.vocab.strings[doc.c[j].morph]))
+                        doc.c[j].morph = self.vocab.morphology.add(extended_morph)
+                    else:
+                        # clobber
+                        doc.c[j].morph = self.vocab.morphology.add(self.cfg["labels_morph"].get(morph, 0))
+                # set POS
+                if doc.c[j].pos == 0 or overwrite:
+                    doc.c[j].pos = self.cfg["labels_pos"].get(morph, 0)
+
+    def get_loss(self, examples, scores):
+        """Find the loss and gradient of loss for the batch of documents and
+        their predicted scores.
+
+        examples (Iterable[Examples]): The batch of examples.
+        scores: Scores representing the model's predictions.
+        RETURNS (Tuple[float, float]): The loss and the gradient.
+
+        DOCS: https://spacy.io/api/morphologizer#get_loss
+        """
+        validate_examples(examples, "Morphologizer.get_loss")
+        loss_func = SequenceCategoricalCrossentropy(names=self.labels, normalize=False)
+        truths = []
+        for eg in examples:
+            eg_truths = []
+            pos_tags = eg.get_aligned("POS", as_string=True)
+            morphs = eg.get_aligned("MORPH", as_string=True)
+            for i in range(len(morphs)):
+                pos = pos_tags[i]
+                morph = morphs[i]
+                # POS may align (same value for multiple tokens) when morph
+                # doesn't, so if either is misaligned (None), treat the
+                # annotation as missing so that truths doesn't end up with an
+                # unknown morph+POS combination
+                if pos is None or morph is None:
+                    label = None
+                # If both are unset, the annotation is missing (empty morph
+                # converted from int is "_" rather than "")
+                elif pos == "" and morph == "":
+                    label = None
+                # Otherwise, generate the combined label
+                else:
+                    label_dict = Morphology.feats_to_dict(morph)
+                    if pos:
+                        label_dict[self.POS_FEAT] = pos
+                    label = self.vocab.strings[self.vocab.morphology.add(label_dict)]
+                    # As a fail-safe, skip any unrecognized labels
+                    if label not in self.labels:
+                        label = None
+                eg_truths.append(label)
+            truths.append(eg_truths)
+        d_scores, loss = loss_func(scores, truths)
+        if self.model.ops.xp.isnan(loss):
+            raise ValueError(Errors.E910.format(name=self.name))
+        return float(loss), d_scores

cgpenv/Lib/site-packages/spacy/pipeline/multitask.pyx

+# cython: infer_types=True, profile=True, binding=True
+from typing import Optional
+import numpy
+from thinc.api import CosineDistance, to_categorical, Model, Config
+from thinc.api import set_dropout_rate
+
+from ..tokens.doc cimport Doc
+
+from .trainable_pipe import TrainablePipe
+from .tagger import Tagger
+from ..training import validate_examples
+from ..language import Language
+from ._parser_internals import nonproj
+from ..attrs import POS, ID
+from ..errors import Errors
+
+
+default_model_config = """
+[model]
+@architectures = "spacy.MultiTask.v1"
+maxout_pieces = 3
+token_vector_width = 96
+
+[model.tok2vec]
+@architectures = "spacy.HashEmbedCNN.v2"
+pretrained_vectors = null
+width = 96
+depth = 4
+embed_size = 2000
+window_size = 1
+maxout_pieces = 2
+subword_features = true
+"""
+DEFAULT_MT_MODEL = Config().from_str(default_model_config)["model"]
+
+
+@Language.factory(
+    "nn_labeller",
+    default_config={"labels": None, "target": "dep_tag_offset", "model": DEFAULT_MT_MODEL}
+)
+def make_nn_labeller(nlp: Language, name: str, model: Model, labels: Optional[dict], target: str):
+    return MultitaskObjective(nlp.vocab, model, name)
+
+
+class MultitaskObjective(Tagger):
+    """Experimental: Assist training of a parser or tagger, by training a
+    side-objective.
+    """
+
+    def __init__(self, vocab, model, name="nn_labeller", *, target):
+        self.vocab = vocab
+        self.model = model
+        self.name = name
+        if target == "dep":
+            self.make_label = self.make_dep
+        elif target == "tag":
+            self.make_label = self.make_tag
+        elif target == "ent":
+            self.make_label = self.make_ent
+        elif target == "dep_tag_offset":
+            self.make_label = self.make_dep_tag_offset
+        elif target == "ent_tag":
+            self.make_label = self.make_ent_tag
+        elif target == "sent_start":
+            self.make_label = self.make_sent_start
+        elif hasattr(target, "__call__"):
+            self.make_label = target
+        else:
+            raise ValueError(Errors.E016)
+        cfg = {"labels": {}, "target": target}
+        self.cfg = dict(cfg)
+
+    @property
+    def labels(self):
+        return self.cfg.setdefault("labels", {})
+
+    @labels.setter
+    def labels(self, value):
+        self.cfg["labels"] = value
+
+    def set_annotations(self, docs, dep_ids):
+        pass
+
+    def initialize(self, get_examples, nlp=None, labels=None):
+        if not hasattr(get_examples, "__call__"):
+            err = Errors.E930.format(name="MultitaskObjective", obj=type(get_examples))
+            raise ValueError(err)
+        if labels is not None:
+            self.labels = labels
+        else:
+            for example in get_examples():
+                for token in example.y:
+                    label = self.make_label(token)
+                    if label is not None and label not in self.labels:
+                        self.labels[label] = len(self.labels)
+        self.model.initialize()   # TODO: fix initialization by defining X and Y
+
+    def predict(self, docs):
+        tokvecs = self.model.get_ref("tok2vec")(docs)
+        scores = self.model.get_ref("softmax")(tokvecs)
+        return tokvecs, scores
+
+    def get_loss(self, examples, scores):
+        cdef int idx = 0
+        correct = numpy.zeros((scores.shape[0],), dtype="i")
+        guesses = scores.argmax(axis=1)
+        docs = [eg.predicted for eg in examples]
+        for i, eg in enumerate(examples):
+            # Handles alignment for tokenization differences
+            doc_annots = eg.get_aligned()  # TODO
+            for j in range(len(eg.predicted)):
+                tok_annots = {key: values[j] for key, values in tok_annots.items()}
+                label = self.make_label(j, tok_annots)
+                if label is None or label not in self.labels:
+                    correct[idx] = guesses[idx]
+                else:
+                    correct[idx] = self.labels[label]
+                idx += 1
+        correct = self.model.ops.xp.array(correct, dtype="i")
+        d_scores = scores - to_categorical(correct, n_classes=scores.shape[1])
+        loss = (d_scores**2).sum()
+        return float(loss), d_scores
+
+    @staticmethod
+    def make_dep(token):
+        return token.dep_
+
+    @staticmethod
+    def make_tag(token):
+        return token.tag_
+
+    @staticmethod
+    def make_ent(token):
+        if token.ent_iob_ == "O":
+            return "O"
+        else:
+            return token.ent_iob_ + "-" + token.ent_type_
+
+    @staticmethod
+    def make_dep_tag_offset(token):
+        dep = token.dep_
+        tag = token.tag_
+        offset = token.head.i - token.i
+        offset = min(offset, 2)
+        offset = max(offset, -2)
+        return f"{dep}-{tag}:{offset}"
+
+    @staticmethod
+    def make_ent_tag(token):
+        if token.ent_iob_ == "O":
+            ent = "O"
+        else:
+            ent = token.ent_iob_ + "-" + token.ent_type_
+        tag = token.tag_
+        return f"{tag}-{ent}"
+
+    @staticmethod
+    def make_sent_start(token):
+        """A multi-task objective for representing sentence boundaries,
+        using BILU scheme. (O is impossible)
+        """
+        if token.is_sent_start and token.is_sent_end:
+            return "U-SENT"
+        elif token.is_sent_start:
+            return "B-SENT"
+        else:
+            return "I-SENT"
+
+
+class ClozeMultitask(TrainablePipe):
+    def __init__(self, vocab, model, **cfg):
+        self.vocab = vocab
+        self.model = model
+        self.cfg = cfg
+        self.distance = CosineDistance(ignore_zeros=True, normalize=False)  # TODO: in config
+
+    def set_annotations(self, docs, dep_ids):
+        pass
+
+    def initialize(self, get_examples, nlp=None):
+        self.model.initialize()  # TODO: fix initialization by defining X and Y
+        X = self.model.ops.alloc((5, self.model.get_ref("tok2vec").get_dim("nO")))
+        self.model.output_layer.initialize(X)
+
+    def predict(self, docs):
+        tokvecs = self.model.get_ref("tok2vec")(docs)
+        vectors = self.model.get_ref("output_layer")(tokvecs)
+        return tokvecs, vectors
+
+    def get_loss(self, examples, vectors, prediction):
+        validate_examples(examples, "ClozeMultitask.get_loss")
+        # The simplest way to implement this would be to vstack the
+        # token.vector values, but that's a bit inefficient, especially on GPU.
+        # Instead we fetch the index into the vectors table for each of our tokens,
+        # and look them up all at once. This prevents data copying.
+        ids = self.model.ops.flatten([eg.predicted.to_array(ID).ravel() for eg in examples])
+        target = vectors[ids]
+        gradient = self.distance.get_grad(prediction, target)
+        loss = self.distance.get_loss(prediction, target)
+        return float(loss), gradient
+
+    def update(self, examples, *, drop=0., sgd=None, losses=None):
+        pass
+
+    def rehearse(self, examples, drop=0., sgd=None, losses=None):
+        if losses is not None and self.name not in losses:
+            losses[self.name] = 0.
+        set_dropout_rate(self.model, drop)
+        validate_examples(examples, "ClozeMultitask.rehearse")
+        docs = [eg.predicted for eg in examples]
+        predictions, bp_predictions = self.model.begin_update()
+        loss, d_predictions = self.get_loss(examples, self.vocab.vectors.data, predictions)
+        bp_predictions(d_predictions)
+        if sgd is not None:
+            self.finish_update(sgd)
+        if losses is not None:
+            losses[self.name] += loss
+        return losses
+
+    def add_label(self, label):
+        raise NotImplementedError

cgpenv/Lib/site-packages/spacy/pipeline/ner.pyx

+# cython: infer_types=True, profile=True, binding=True
+from collections import defaultdict
+from typing import Optional, Iterable, Callable
+from thinc.api import Model, Config
+
+from ._parser_internals.transition_system import TransitionSystem
+from .transition_parser cimport Parser
+from ._parser_internals.ner cimport BiluoPushDown
+
+from ..language import Language
+from ..scorer import get_ner_prf, PRFScore
+from ..util import registry
+
+
+default_model_config = """
+[model]
+@architectures = "spacy.TransitionBasedParser.v2"
+state_type = "ner"
+extra_state_tokens = false
+hidden_width = 64
+maxout_pieces = 2
+use_upper = true
+
+[model.tok2vec]
+@architectures = "spacy.HashEmbedCNN.v2"
+pretrained_vectors = null
+width = 96
+depth = 4
+embed_size = 2000
+window_size = 1
+maxout_pieces = 3
+subword_features = true
+"""
+DEFAULT_NER_MODEL = Config().from_str(default_model_config)["model"]
+
+
+@Language.factory(
+    "ner",
+    assigns=["doc.ents", "token.ent_iob", "token.ent_type"],
+    default_config={
+        "moves": None,
+        "update_with_oracle_cut_size": 100,
+        "model": DEFAULT_NER_MODEL,
+        "incorrect_spans_key": None,
+        "scorer": {"@scorers": "spacy.ner_scorer.v1"},
+    },
+    default_score_weights={"ents_f": 1.0, "ents_p": 0.0, "ents_r": 0.0, "ents_per_type": None},
+
+)
+def make_ner(
+    nlp: Language,
+    name: str,
+    model: Model,
+    moves: Optional[TransitionSystem],
+    update_with_oracle_cut_size: int,
+    incorrect_spans_key: Optional[str],
+    scorer: Optional[Callable],
+):
+    """Create a transition-based EntityRecognizer component. The entity recognizer
+    identifies non-overlapping labelled spans of tokens.
+
+    The transition-based algorithm used encodes certain assumptions that are
+    effective for "traditional" named entity recognition tasks, but may not be
+    a good fit for every span identification problem. Specifically, the loss
+    function optimizes for whole entity accuracy, so if your inter-annotator
+    agreement on boundary tokens is low, the component will likely perform poorly
+    on your problem. The transition-based algorithm also assumes that the most
+    decisive information about your entities will be close to their initial tokens.
+    If your entities are long and characterised by tokens in their middle, the
+    component will likely do poorly on your task.
+
+    model (Model): The model for the transition-based parser. The model needs
+        to have a specific substructure of named components --- see the
+        spacy.ml.tb_framework.TransitionModel for details.
+    moves (Optional[TransitionSystem]): This defines how the parse-state is created,
+        updated and evaluated. If 'moves' is None, a new instance is
+        created with `self.TransitionSystem()`. Defaults to `None`.
+    update_with_oracle_cut_size (int): During training, cut long sequences into
+        shorter segments by creating intermediate states based on the gold-standard
+        history. The model is not very sensitive to this parameter, so you usually
+        won't need to change it. 100 is a good default.
+    incorrect_spans_key (Optional[str]): Identifies spans that are known
+        to be incorrect entity annotations. The incorrect entity annotations
+        can be stored in the span group, under this key.
+    scorer (Optional[Callable]): The scoring method.
+    """
+    return EntityRecognizer(
+        nlp.vocab,
+        model,
+        name,
+        moves=moves,
+        update_with_oracle_cut_size=update_with_oracle_cut_size,
+        incorrect_spans_key=incorrect_spans_key,
+        multitasks=[],
+        beam_width=1,
+        beam_density=0.0,
+        beam_update_prob=0.0,
+        scorer=scorer,
+    )
+
+@Language.factory(
+    "beam_ner",
+    assigns=["doc.ents", "token.ent_iob", "token.ent_type"],
+    default_config={
+        "moves": None,
+        "update_with_oracle_cut_size": 100,
+        "model": DEFAULT_NER_MODEL,
+        "beam_density": 0.01,
+        "beam_update_prob": 0.5,
+        "beam_width": 32,
+        "incorrect_spans_key": None,
+        "scorer": None,
+    },
+    default_score_weights={"ents_f": 1.0, "ents_p": 0.0, "ents_r": 0.0, "ents_per_type": None},
+)
+def make_beam_ner(
+    nlp: Language,
+    name: str,
+    model: Model,
+    moves: Optional[TransitionSystem],
+    update_with_oracle_cut_size: int,
+    beam_width: int,
+    beam_density: float,
+    beam_update_prob: float,
+    incorrect_spans_key: Optional[str],
+    scorer: Optional[Callable],
+):
+    """Create a transition-based EntityRecognizer component that uses beam-search.
+    The entity recognizer identifies non-overlapping labelled spans of tokens.
+
+    The transition-based algorithm used encodes certain assumptions that are
+    effective for "traditional" named entity recognition tasks, but may not be
+    a good fit for every span identification problem. Specifically, the loss
+    function optimizes for whole entity accuracy, so if your inter-annotator
+    agreement on boundary tokens is low, the component will likely perform poorly
+    on your problem. The transition-based algorithm also assumes that the most
+    decisive information about your entities will be close to their initial tokens.
+    If your entities are long and characterised by tokens in their middle, the
+    component will likely do poorly on your task.
+
+    model (Model): The model for the transition-based parser. The model needs
+        to have a specific substructure of named components --- see the
+        spacy.ml.tb_framework.TransitionModel for details.
+    moves (Optional[TransitionSystem]): This defines how the parse-state is created,
+        updated and evaluated. If 'moves' is None, a new instance is
+        created with `self.TransitionSystem()`. Defaults to `None`.
+    update_with_oracle_cut_size (int): During training, cut long sequences into
+        shorter segments by creating intermediate states based on the gold-standard
+        history. The model is not very sensitive to this parameter, so you usually
+        won't need to change it. 100 is a good default.
+    beam_width (int): The number of candidate analyses to maintain.
+    beam_density (float): The minimum ratio between the scores of the first and
+        last candidates in the beam. This allows the parser to avoid exploring
+        candidates that are too far behind. This is mostly intended to improve
+        efficiency, but it can also improve accuracy as deeper search is not
+        always better.
+    beam_update_prob (float): The chance of making a beam update, instead of a
+        greedy update. Greedy updates are an approximation for the beam updates,
+        and are faster to compute.
+    incorrect_spans_key (Optional[str]): Optional key into span groups of
+        entities known to be non-entities.
+    scorer (Optional[Callable]): The scoring method.
+    """
+    return EntityRecognizer(
+        nlp.vocab,
+        model,
+        name,
+        moves=moves,
+        update_with_oracle_cut_size=update_with_oracle_cut_size,
+        multitasks=[],
+        beam_width=beam_width,
+        beam_density=beam_density,
+        beam_update_prob=beam_update_prob,
+        incorrect_spans_key=incorrect_spans_key,
+        scorer=scorer,
+    )
+
+
+def ner_score(examples, **kwargs):
+    return get_ner_prf(examples, **kwargs)
+
+
+@registry.scorers("spacy.ner_scorer.v1")
+def make_ner_scorer():
+    return ner_score
+
+
+cdef class EntityRecognizer(Parser):
+    """Pipeline component for named entity recognition.
+
+    DOCS: https://spacy.io/api/entityrecognizer
+    """
+    TransitionSystem = BiluoPushDown
+
+    def __init__(
+        self,
+        vocab,
+        model,
+        name="ner",
+        moves=None,
+        *,
+        update_with_oracle_cut_size=100,
+        beam_width=1,
+        beam_density=0.0,
+        beam_update_prob=0.0,
+        multitasks=tuple(),
+        incorrect_spans_key=None,
+        scorer=ner_score,
+    ):
+        """Create an EntityRecognizer.
+        """
+        super().__init__(
+            vocab,
+            model,
+            name,
+            moves,
+            update_with_oracle_cut_size=update_with_oracle_cut_size,
+            min_action_freq=1,   # not relevant for NER
+            learn_tokens=False,  # not relevant for NER
+            beam_width=beam_width,
+            beam_density=beam_density,
+            beam_update_prob=beam_update_prob,
+            multitasks=multitasks,
+            incorrect_spans_key=incorrect_spans_key,
+            scorer=scorer,
+        )
+
+    def add_multitask_objective(self, mt_component):
+        """Register another component as a multi-task objective. Experimental."""
+        self._multitasks.append(mt_component)
+
+    def init_multitask_objectives(self, get_examples, nlp=None, **cfg):
+        """Setup multi-task objective components. Experimental and internal."""
+        # TODO: transfer self.model.get_ref("tok2vec") to the multitask's model ?
+        for labeller in self._multitasks:
+            labeller.model.set_dim("nO", len(self.labels))
+            if labeller.model.has_ref("output_layer"):
+                labeller.model.get_ref("output_layer").set_dim("nO", len(self.labels))
+            labeller.initialize(get_examples, nlp=nlp)
+
+    @property
+    def labels(self):
+        # Get the labels from the model by looking at the available moves, e.g.
+        # B-PERSON, I-PERSON, L-PERSON, U-PERSON
+        labels = set(move.split("-")[1] for move in self.move_names
+                     if move[0] in ("B", "I", "L", "U"))
+        return tuple(sorted(labels))
+
+    def scored_ents(self, beams):
+        """Return a dictionary of (start, end, label) tuples with corresponding scores
+        for each beam/doc that was processed.
+        """
+        entity_scores = []
+        for beam in beams:
+            score_dict = defaultdict(float)
+            for score, ents in self.moves.get_beam_parses(beam):
+                for start, end, label in ents:
+                    score_dict[(start, end, label)] += score
+            entity_scores.append(score_dict)
+        return entity_scores

cgpenv/Lib/site-packages/spacy/pipeline/pipe.pxd

+cdef class Pipe:
+    cdef public str name

cgpenv/Lib/site-packages/spacy/pipeline/pipe.pyi

+from pathlib import Path
+from typing import Any, Callable, Dict, Iterable, Iterator, List
+from typing import NoReturn, Optional, Tuple, Union
+
+from ..tokens.doc import Doc
+
+from ..training import Example
+from ..language import Language
+
+class Pipe:
+    def __call__(self, doc: Doc) -> Doc: ...
+    def pipe(
+        self, stream: Iterable[Doc], *, batch_size: int = ...
+    ) -> Iterator[Doc]: ...
+    def initialize(
+        self,
+        get_examples: Callable[[], Iterable[Example]],
+        *,
+        nlp: Language = ...,
+    ) -> None: ...
+    def score(
+        self, examples: Iterable[Example], **kwargs: Any
+    ) -> Dict[str, Union[float, Dict[str, float]]]: ...
+    @property
+    def is_trainable(self) -> bool: ...
+    @property
+    def labels(self) -> Tuple[str, ...]: ...
+    @property
+    def label_data(self) -> Any: ...
+    def _require_labels(self) -> None: ...
+    def set_error_handler(
+        self, error_handler: Callable[[str, "Pipe", List[Doc], Exception], NoReturn]
+    ) -> None: ...
+    def get_error_handler(
+        self,
+    ) -> Callable[[str, "Pipe", List[Doc], Exception], NoReturn]: ...
+
+def deserialize_config(path: Path) -> Any: ...

cgpenv/Lib/site-packages/spacy/pipeline/pipe.pyx

+# cython: infer_types=True, profile=True
+from typing import Optional, Tuple, Iterable, Iterator, Callable, Union, Dict
+import srsly
+import warnings
+
+from ..tokens.doc cimport Doc
+
+from ..training import Example
+from ..errors import Errors, Warnings
+from ..language import Language
+from ..util import raise_error
+
+cdef class Pipe:
+    """This class is a base class and not instantiated directly. It provides
+    an interface for pipeline components to implement.
+    Trainable pipeline components like the EntityRecognizer or TextCategorizer
+    should inherit from the subclass 'TrainablePipe'.
+
+    DOCS: https://spacy.io/api/pipe
+    """
+
+    @classmethod
+    def __init_subclass__(cls, **kwargs):
+        """Raise a warning if an inheriting class implements 'begin_training'
+         (from v2) instead of the new 'initialize' method (from v3)"""
+        if hasattr(cls, "begin_training"):
+            warnings.warn(Warnings.W088.format(name=cls.__name__))
+
+    def __call__(self, Doc doc) -> Doc:
+        """Apply the pipe to one document. The document is modified in place,
+        and returned. This usually happens under the hood when the nlp object
+        is called on a text and all components are applied to the Doc.
+
+        docs (Doc): The Doc to process.
+        RETURNS (Doc): The processed Doc.
+
+        DOCS: https://spacy.io/api/pipe#call
+        """
+        raise NotImplementedError(Errors.E931.format(parent="Pipe", method="__call__", name=self.name))
+
+    def pipe(self, stream: Iterable[Doc], *, batch_size: int=128) -> Iterator[Doc]:
+        """Apply the pipe to a stream of documents. This usually happens under
+        the hood when the nlp object is called on a text and all components are
+        applied to the Doc.
+
+        stream (Iterable[Doc]): A stream of documents.
+        batch_size (int): The number of documents to buffer.
+        YIELDS (Doc): Processed documents in order.
+
+        DOCS: https://spacy.io/api/pipe#pipe
+        """
+        error_handler = self.get_error_handler()
+        for doc in stream:
+            try:
+                doc = self(doc)
+                yield doc
+            except Exception as e:
+                error_handler(self.name, self, [doc], e)
+
+    def initialize(self, get_examples: Callable[[], Iterable[Example]], *, nlp: Language=None):
+        """Initialize the pipe. For non-trainable components, this method
+        is optional. For trainable components, which should inherit
+        from the subclass TrainablePipe, the provided data examples
+        should be used to ensure that the internal model is initialized
+        properly and all input/output dimensions throughout the network are
+        inferred.
+
+        get_examples (Callable[[], Iterable[Example]]): Function that
+            returns a representative sample of gold-standard Example objects.
+        nlp (Language): The current nlp object the component is part of.
+
+        DOCS: https://spacy.io/api/pipe#initialize
+        """
+        pass
+
+    def score(self, examples: Iterable[Example], **kwargs) -> Dict[str, Union[float, Dict[str, float]]]:
+        """Score a batch of examples.
+
+        examples (Iterable[Example]): The examples to score.
+        RETURNS (Dict[str, Any]): The scores.
+
+        DOCS: https://spacy.io/api/pipe#score
+        """
+        if hasattr(self, "scorer") and self.scorer is not None:
+            scorer_kwargs = {}
+            # use default settings from cfg (e.g., threshold)
+            if hasattr(self, "cfg") and isinstance(self.cfg, dict):
+                scorer_kwargs.update(self.cfg)
+            # override self.cfg["labels"] with self.labels
+            if hasattr(self, "labels"):
+                scorer_kwargs["labels"] = self.labels
+            # override with kwargs settings
+            scorer_kwargs.update(kwargs)
+            return self.scorer(examples, **scorer_kwargs)
+        return {}
+
+    @property
+    def is_trainable(self) -> bool:
+        return False
+
+    @property
+    def labels(self) -> Tuple[str, ...]:
+        return tuple()
+
+    @property
+    def label_data(self):
+        """Optional JSON-serializable data that would be sufficient to recreate
+        the label set if provided to the `pipe.initialize()` method.
+        """
+        return None
+
+    def _require_labels(self) -> None:
+        """Raise an error if this component has no labels defined."""
+        if not self.labels or list(self.labels) == [""]:
+            raise ValueError(Errors.E143.format(name=self.name))
+
+    def set_error_handler(self, error_handler: Callable) -> None:
+        """Set an error handler function.
+
+        error_handler (Callable[[str, Callable[[Doc], Doc], List[Doc], Exception], None]):
+            Function that deals with a failing batch of documents. This callable function should take in
+            the component's name, the component itself, the offending batch of documents, and the exception
+            that was thrown.
+
+        DOCS: https://spacy.io/api/pipe#set_error_handler
+        """
+        self.error_handler = error_handler
+
+    def get_error_handler(self) -> Callable:
+        """Retrieve the error handler function.
+
+        RETURNS (Callable): The error handler, or if it's not set a default function that just reraises.
+
+        DOCS: https://spacy.io/api/pipe#get_error_handler
+        """
+        if hasattr(self, "error_handler"):
+            return self.error_handler
+        return raise_error
+
+
+def deserialize_config(path):
+    if path.exists():
+        return srsly.read_json(path)
+    else:
+        return {}

cgpenv/Lib/site-packages/spacy/pipeline/sentencizer.pyx

+# cython: infer_types=True, profile=True, binding=True
+from typing import Optional, List, Callable
+import srsly
+
+from ..tokens.doc cimport Doc
+
+from .pipe import Pipe
+from .senter import senter_score
+from ..language import Language
+from ..scorer import Scorer
+from .. import util
+
+# see #9050
+BACKWARD_OVERWRITE = False
+
+@Language.factory(
+    "sentencizer",
+    assigns=["token.is_sent_start", "doc.sents"],
+    default_config={"punct_chars": None, "overwrite": False, "scorer": {"@scorers": "spacy.senter_scorer.v1"}},
+    default_score_weights={"sents_f": 1.0, "sents_p": 0.0, "sents_r": 0.0},
+)
+def make_sentencizer(
+    nlp: Language,
+    name: str,
+    punct_chars: Optional[List[str]],
+    overwrite: bool,
+    scorer: Optional[Callable],
+):
+    return Sentencizer(name, punct_chars=punct_chars, overwrite=overwrite, scorer=scorer)
+
+
+class Sentencizer(Pipe):
+    """Segment the Doc into sentences using a rule-based strategy.
+
+    DOCS: https://spacy.io/api/sentencizer
+    """
+
+    default_punct_chars = ['!', '.', '?', '։', '؟', '۔', '܀', '܁', '܂', '߹',
+            '।', '॥', '၊', '။', '።', '፧', '፨', '᙮', '᜵', '᜶', '᠃', '᠉', '᥄',
+            '᥅', '᪨', '᪩', '᪪', '᪫', '᭚', '᭛', '᭞', '᭟', '᰻', '᰼', '᱾', '᱿',
+            '‼', '‽', '⁇', '⁈', '⁉', '⸮', '⸼', '꓿', '꘎', '꘏', '꛳', '꛷', '꡶',
+            '꡷', '꣎', '꣏', '꤯', '꧈', '꧉', '꩝', '꩞', '꩟', '꫰', '꫱', '꯫', '﹒',
+            '﹖', '﹗', '！', '．', '？', '𐩖', '𐩗', '𑁇', '𑁈', '𑂾', '𑂿', '𑃀',
+            '𑃁', '𑅁', '𑅂', '𑅃', '𑇅', '𑇆', '𑇍', '𑇞', '𑇟', '𑈸', '𑈹', '𑈻', '𑈼',
+            '𑊩', '𑑋', '𑑌', '𑗂', '𑗃', '𑗉', '𑗊', '𑗋', '𑗌', '𑗍', '𑗎', '𑗏', '𑗐',
+            '𑗑', '𑗒', '𑗓', '𑗔', '𑗕', '𑗖', '𑗗', '𑙁', '𑙂', '𑜼', '𑜽', '𑜾', '𑩂',
+            '𑩃', '𑪛', '𑪜', '𑱁', '𑱂', '𖩮', '𖩯', '𖫵', '𖬷', '𖬸', '𖭄', '𛲟', '𝪈',
+            '｡', '。']
+
+    def __init__(
+        self,
+        name="sentencizer",
+        *,
+        punct_chars=None,
+        overwrite=BACKWARD_OVERWRITE,
+        scorer=senter_score,
+    ):
+        """Initialize the sentencizer.
+
+        punct_chars (list): Punctuation characters to split on. Will be
+            serialized with the nlp object.
+        scorer (Optional[Callable]): The scoring method. Defaults to
+            Scorer.score_spans for the attribute "sents".
+
+        DOCS: https://spacy.io/api/sentencizer#init
+        """
+        self.name = name
+        if punct_chars:
+            self.punct_chars = set(punct_chars)
+        else:
+            self.punct_chars = set(self.default_punct_chars)
+        self.overwrite = overwrite
+        self.scorer = scorer
+
+    def __call__(self, doc):
+        """Apply the sentencizer to a Doc and set Token.is_sent_start.
+
+        doc (Doc): The document to process.
+        RETURNS (Doc): The processed Doc.
+
+        DOCS: https://spacy.io/api/sentencizer#call
+        """
+        error_handler = self.get_error_handler()
+        try:
+            tags = self.predict([doc])
+            self.set_annotations([doc], tags)
+            return doc
+        except Exception as e:
+            error_handler(self.name, self, [doc], e)
+
+    def predict(self, docs):
+        """Apply the pipe to a batch of docs, without modifying them.
+
+        docs (Iterable[Doc]): The documents to predict.
+        RETURNS: The predictions for each document.
+        """
+        if not any(len(doc) for doc in docs):
+            # Handle cases where there are no tokens in any docs.
+            guesses = [[] for doc in docs]
+            return guesses
+        guesses = []
+        for doc in docs:
+            doc_guesses = [False] * len(doc)
+            if len(doc) > 0:
+                start = 0
+                seen_period = False
+                doc_guesses[0] = True
+                for i, token in enumerate(doc):
+                    is_in_punct_chars = token.text in self.punct_chars
+                    if seen_period and not token.is_punct and not is_in_punct_chars:
+                        doc_guesses[start] = True
+                        start = token.i
+                        seen_period = False
+                    elif is_in_punct_chars:
+                        seen_period = True
+                if start < len(doc):
+                    doc_guesses[start] = True
+            guesses.append(doc_guesses)
+        return guesses
+
+    def set_annotations(self, docs, batch_tag_ids):
+        """Modify a batch of documents, using pre-computed scores.
+
+        docs (Iterable[Doc]): The documents to modify.
+        scores: The tag IDs produced by Sentencizer.predict.
+        """
+        if isinstance(docs, Doc):
+            docs = [docs]
+        cdef Doc doc
+        cdef int idx = 0
+        for i, doc in enumerate(docs):
+            doc_tag_ids = batch_tag_ids[i]
+            for j, tag_id in enumerate(doc_tag_ids):
+                if doc.c[j].sent_start == 0 or self.overwrite:
+                    if tag_id:
+                        doc.c[j].sent_start = 1
+                    else:
+                        doc.c[j].sent_start = -1
+
+    def to_bytes(self, *, exclude=tuple()):
+        """Serialize the sentencizer to a bytestring.
+
+        RETURNS (bytes): The serialized object.
+
+        DOCS: https://spacy.io/api/sentencizer#to_bytes
+        """
+        return srsly.msgpack_dumps({"punct_chars": list(self.punct_chars), "overwrite": self.overwrite})
+
+    def from_bytes(self, bytes_data, *, exclude=tuple()):
+        """Load the sentencizer from a bytestring.
+
+        bytes_data (bytes): The data to load.
+        returns (Sentencizer): The loaded object.
+
+        DOCS: https://spacy.io/api/sentencizer#from_bytes
+        """
+        cfg = srsly.msgpack_loads(bytes_data)
+        self.punct_chars = set(cfg.get("punct_chars", self.default_punct_chars))
+        self.overwrite = cfg.get("overwrite", self.overwrite)
+        return self
+
+    def to_disk(self, path, *, exclude=tuple()):
+        """Serialize the sentencizer to disk.
+
+        DOCS: https://spacy.io/api/sentencizer#to_disk
+        """
+        path = util.ensure_path(path)
+        path = path.with_suffix(".json")
+        srsly.write_json(path, {"punct_chars": list(self.punct_chars), "overwrite": self.overwrite})
+
+
+    def from_disk(self, path, *, exclude=tuple()):
+        """Load the sentencizer from disk.
+
+        DOCS: https://spacy.io/api/sentencizer#from_disk
+        """
+        path = util.ensure_path(path)
+        path = path.with_suffix(".json")
+        cfg = srsly.read_json(path)
+        self.punct_chars = set(cfg.get("punct_chars", self.default_punct_chars))
+        self.overwrite = cfg.get("overwrite", self.overwrite)
+        return self

cgpenv/Lib/site-packages/spacy/pipeline/senter.pyx

+# cython: infer_types=True, profile=True, binding=True
+from itertools import islice
+from typing import Optional, Callable
+
+import srsly
+from thinc.api import Model, SequenceCategoricalCrossentropy, Config
+
+from ..tokens.doc cimport Doc
+
+from .tagger import Tagger
+from ..language import Language
+from ..errors import Errors
+from ..scorer import Scorer
+from ..training import validate_examples, validate_get_examples
+from ..util import registry
+from .. import util
+
+# See #9050
+BACKWARD_OVERWRITE = False
+
+default_model_config = """
+[model]
+@architectures = "spacy.Tagger.v1"
+
+[model.tok2vec]
+@architectures = "spacy.HashEmbedCNN.v2"
+pretrained_vectors = null
+width = 12
+depth = 1
+embed_size = 2000
+window_size = 1
+maxout_pieces = 2
+subword_features = true
+"""
+DEFAULT_SENTER_MODEL = Config().from_str(default_model_config)["model"]
+
+
+@Language.factory(
+    "senter",
+    assigns=["token.is_sent_start"],
+    default_config={"model": DEFAULT_SENTER_MODEL, "overwrite": False, "scorer": {"@scorers": "spacy.senter_scorer.v1"}},
+    default_score_weights={"sents_f": 1.0, "sents_p": 0.0, "sents_r": 0.0},
+)
+def make_senter(nlp: Language, name: str, model: Model, overwrite: bool, scorer: Optional[Callable]):
+    return SentenceRecognizer(nlp.vocab, model, name, overwrite=overwrite, scorer=scorer)
+
+
+def senter_score(examples, **kwargs):
+    def has_sents(doc):
+        return doc.has_annotation("SENT_START")
+
+    results = Scorer.score_spans(examples, "sents", has_annotation=has_sents, **kwargs)
+    del results["sents_per_type"]
+    return results
+
+
+@registry.scorers("spacy.senter_scorer.v1")
+def make_senter_scorer():
+    return senter_score
+
+
+class SentenceRecognizer(Tagger):
+    """Pipeline component for sentence segmentation.
+
+    DOCS: https://spacy.io/api/sentencerecognizer
+    """
+    def __init__(
+        self,
+        vocab,
+        model,
+        name="senter",
+        *,
+        overwrite=BACKWARD_OVERWRITE,
+        scorer=senter_score,
+    ):
+        """Initialize a sentence recognizer.
+
+        vocab (Vocab): The shared vocabulary.
+        model (thinc.api.Model): The Thinc Model powering the pipeline component.
+        name (str): The component instance name, used to add entries to the
+            losses during training.
+        scorer (Optional[Callable]): The scoring method. Defaults to
+            Scorer.score_spans for the attribute "sents".
+
+        DOCS: https://spacy.io/api/sentencerecognizer#init
+        """
+        self.vocab = vocab
+        self.model = model
+        self.name = name
+        self._rehearsal_model = None
+        self.cfg = {"overwrite": overwrite}
+        self.scorer = scorer
+
+    @property
+    def labels(self):
+        """RETURNS (Tuple[str]): The labels."""
+        # labels are numbered by index internally, so this matches GoldParse
+        # and Example where the sentence-initial tag is 1 and other positions
+        # are 0
+        return tuple(["I", "S"])
+
+    @property
+    def label_data(self):
+        return None
+
+    def set_annotations(self, docs, batch_tag_ids):
+        """Modify a batch of documents, using pre-computed scores.
+
+        docs (Iterable[Doc]): The documents to modify.
+        batch_tag_ids: The IDs to set, produced by SentenceRecognizer.predict.
+
+        DOCS: https://spacy.io/api/sentencerecognizer#set_annotations
+        """
+        if isinstance(docs, Doc):
+            docs = [docs]
+        cdef Doc doc
+        cdef bint overwrite = self.cfg["overwrite"]
+        for i, doc in enumerate(docs):
+            doc_tag_ids = batch_tag_ids[i]
+            if hasattr(doc_tag_ids, "get"):
+                doc_tag_ids = doc_tag_ids.get()
+            for j, tag_id in enumerate(doc_tag_ids):
+                if doc.c[j].sent_start == 0 or overwrite:
+                    if tag_id == 1:
+                        doc.c[j].sent_start = 1
+                    else:
+                        doc.c[j].sent_start = -1
+
+    def get_loss(self, examples, scores):
+        """Find the loss and gradient of loss for the batch of documents and
+        their predicted scores.
+
+        examples (Iterable[Examples]): The batch of examples.
+        scores: Scores representing the model's predictions.
+        RETURNS (Tuple[float, float]): The loss and the gradient.
+
+        DOCS: https://spacy.io/api/sentencerecognizer#get_loss
+        """
+        validate_examples(examples, "SentenceRecognizer.get_loss")
+        labels = self.labels
+        loss_func = SequenceCategoricalCrossentropy(names=labels, normalize=False)
+        truths = []
+        for eg in examples:
+            eg_truth = []
+            for x in eg.get_aligned("SENT_START"):
+                if x is None:
+                    eg_truth.append(None)
+                elif x == 1:
+                    eg_truth.append(labels[1])
+                else:
+                    # anything other than 1: 0, -1, -1 as uint64
+                    eg_truth.append(labels[0])
+            truths.append(eg_truth)
+        d_scores, loss = loss_func(scores, truths)
+        if self.model.ops.xp.isnan(loss):
+            raise ValueError(Errors.E910.format(name=self.name))
+        return float(loss), d_scores
+
+    def initialize(self, get_examples, *, nlp=None):
+        """Initialize the pipe for training, using a representative set
+        of data examples.
+
+        get_examples (Callable[[], Iterable[Example]]): Function that
+            returns a representative sample of gold-standard Example objects.
+        nlp (Language): The current nlp object the component is part of.
+
+        DOCS: https://spacy.io/api/sentencerecognizer#initialize
+        """
+        validate_get_examples(get_examples, "SentenceRecognizer.initialize")
+        util.check_lexeme_norms(self.vocab, "senter")
+        doc_sample = []
+        label_sample = []
+        assert self.labels, Errors.E924.format(name=self.name)
+        for example in islice(get_examples(), 10):
+            doc_sample.append(example.x)
+            gold_tags = example.get_aligned("SENT_START")
+            gold_array = [[1.0 if tag == gold_tag else 0.0 for tag in self.labels] for gold_tag in gold_tags]
+            label_sample.append(self.model.ops.asarray(gold_array, dtype="float32"))
+        assert len(doc_sample) > 0, Errors.E923.format(name=self.name)
+        assert len(label_sample) > 0, Errors.E923.format(name=self.name)
+        self.model.initialize(X=doc_sample, Y=label_sample)
+
+    def add_label(self, label, values=None):
+        raise NotImplementedError

cgpenv/Lib/site-packages/spacy/pipeline/spancat.py

+import numpy
+from typing import List, Dict, Callable, Tuple, Optional, Iterable, Any, cast
+from thinc.api import Config, Model, get_current_ops, set_dropout_rate, Ops
+from thinc.api import Optimizer
+from thinc.types import Ragged, Ints2d, Floats2d, Ints1d
+
+from ..compat import Protocol, runtime_checkable
+from ..scorer import Scorer
+from ..language import Language
+from .trainable_pipe import TrainablePipe
+from ..tokens import Doc, SpanGroup, Span
+from ..vocab import Vocab
+from ..training import Example, validate_examples
+from ..errors import Errors
+from ..util import registry
+
+
+spancat_default_config = """
+[model]
+@architectures = "spacy.SpanCategorizer.v1"
+scorer = {"@layers": "spacy.LinearLogistic.v1"}
+
+[model.reducer]
+@layers = spacy.mean_max_reducer.v1
+hidden_size = 128
+
+[model.tok2vec]
+@architectures = "spacy.Tok2Vec.v1"
+
+[model.tok2vec.embed]
+@architectures = "spacy.MultiHashEmbed.v1"
+width = 96
+rows = [5000, 2000, 1000, 1000]
+attrs = ["ORTH", "PREFIX", "SUFFIX", "SHAPE"]
+include_static_vectors = false
+
+[model.tok2vec.encode]
+@architectures = "spacy.MaxoutWindowEncoder.v1"
+width = ${model.tok2vec.embed.width}
+window_size = 1
+maxout_pieces = 3
+depth = 4
+"""
+
+DEFAULT_SPANCAT_MODEL = Config().from_str(spancat_default_config)["model"]
+
+
+@runtime_checkable
+class Suggester(Protocol):
+    def __call__(self, docs: Iterable[Doc], *, ops: Optional[Ops] = None) -> Ragged:
+        ...
+
+
+@registry.misc("spacy.ngram_suggester.v1")
+def build_ngram_suggester(sizes: List[int]) -> Suggester:
+    """Suggest all spans of the given lengths. Spans are returned as a ragged
+    array of integers. The array has two columns, indicating the start and end
+    position."""
+
+    def ngram_suggester(docs: Iterable[Doc], *, ops: Optional[Ops] = None) -> Ragged:
+        if ops is None:
+            ops = get_current_ops()
+        spans = []
+        lengths = []
+        for doc in docs:
+            starts = ops.xp.arange(len(doc), dtype="i")
+            starts = starts.reshape((-1, 1))
+            length = 0
+            for size in sizes:
+                if size <= len(doc):
+                    starts_size = starts[: len(doc) - (size - 1)]
+                    spans.append(ops.xp.hstack((starts_size, starts_size + size)))
+                    length += spans[-1].shape[0]
+                if spans:
+                    assert spans[-1].ndim == 2, spans[-1].shape
+            lengths.append(length)
+        lengths_array = cast(Ints1d, ops.asarray(lengths, dtype="i"))
+        if len(spans) > 0:
+            output = Ragged(ops.xp.vstack(spans), lengths_array)
+        else:
+            output = Ragged(ops.xp.zeros((0, 0)), lengths_array)
+
+        assert output.dataXd.ndim == 2
+        return output
+
+    return ngram_suggester
+
+
+@registry.misc("spacy.ngram_range_suggester.v1")
+def build_ngram_range_suggester(min_size: int, max_size: int) -> Suggester:
+    """Suggest all spans of the given lengths between a given min and max value - both inclusive.
+    Spans are returned as a ragged array of integers. The array has two columns,
+    indicating the start and end position."""
+    sizes = list(range(min_size, max_size + 1))
+    return build_ngram_suggester(sizes)
+
+
+@Language.factory(
+    "spancat",
+    assigns=["doc.spans"],
+    default_config={
+        "threshold": 0.5,
+        "spans_key": "sc",
+        "max_positive": None,
+        "model": DEFAULT_SPANCAT_MODEL,
+        "suggester": {"@misc": "spacy.ngram_suggester.v1", "sizes": [1, 2, 3]},
+        "scorer": {"@scorers": "spacy.spancat_scorer.v1"},
+    },
+    default_score_weights={"spans_sc_f": 1.0, "spans_sc_p": 0.0, "spans_sc_r": 0.0},
+)
+def make_spancat(
+    nlp: Language,
+    name: str,
+    suggester: Suggester,
+    model: Model[Tuple[List[Doc], Ragged], Floats2d],
+    spans_key: str,
+    scorer: Optional[Callable],
+    threshold: float,
+    max_positive: Optional[int],
+) -> "SpanCategorizer":
+    """Create a SpanCategorizer component. The span categorizer consists of two
+    parts: a suggester function that proposes candidate spans, and a labeller
+    model that predicts one or more labels for each span.
+
+    suggester (Callable[[Iterable[Doc], Optional[Ops]], Ragged]): A function that suggests spans.
+        Spans are returned as a ragged array with two integer columns, for the
+        start and end positions.
+    model (Model[Tuple[List[Doc], Ragged], Floats2d]): A model instance that
+        is given a list of documents and (start, end) indices representing
+        candidate span offsets. The model predicts a probability for each category
+        for each span.
+    spans_key (str): Key of the doc.spans dict to save the spans under. During
+        initialization and training, the component will look for spans on the
+        reference document under the same key.
+    threshold (float): Minimum probability to consider a prediction positive.
+        Spans with a positive prediction will be saved on the Doc. Defaults to
+        0.5.
+    max_positive (Optional[int]): Maximum number of labels to consider positive
+        per span. Defaults to None, indicating no limit.
+    """
+    return SpanCategorizer(
+        nlp.vocab,
+        suggester=suggester,
+        model=model,
+        spans_key=spans_key,
+        threshold=threshold,
+        max_positive=max_positive,
+        name=name,
+        scorer=scorer,
+    )
+
+
+def spancat_score(examples: Iterable[Example], **kwargs) -> Dict[str, Any]:
+    kwargs = dict(kwargs)
+    attr_prefix = "spans_"
+    key = kwargs["spans_key"]
+    kwargs.setdefault("attr", f"{attr_prefix}{key}")
+    kwargs.setdefault("allow_overlap", True)
+    kwargs.setdefault(
+        "getter", lambda doc, key: doc.spans.get(key[len(attr_prefix) :], [])
+    )
+    kwargs.setdefault("has_annotation", lambda doc: key in doc.spans)
+    return Scorer.score_spans(examples, **kwargs)
+
+
+@registry.scorers("spacy.spancat_scorer.v1")
+def make_spancat_scorer():
+    return spancat_score
+
+
+class SpanCategorizer(TrainablePipe):
+    """Pipeline component to label spans of text.
+
+    DOCS: https://spacy.io/api/spancategorizer
+    """
+
+    def __init__(
+        self,
+        vocab: Vocab,
+        model: Model[Tuple[List[Doc], Ragged], Floats2d],
+        suggester: Suggester,
+        name: str = "spancat",
+        *,
+        spans_key: str = "spans",
+        threshold: float = 0.5,
+        max_positive: Optional[int] = None,
+        scorer: Optional[Callable] = spancat_score,
+    ) -> None:
+        """Initialize the span categorizer.
+        vocab (Vocab): The shared vocabulary.
+        model (thinc.api.Model): The Thinc Model powering the pipeline component.
+        name (str): The component instance name, used to add entries to the
+            losses during training.
+        spans_key (str): Key of the Doc.spans dict to save the spans under.
+            During initialization and training, the component will look for
+            spans on the reference document under the same key. Defaults to
+            `"spans"`.
+        threshold (float): Minimum probability to consider a prediction
+            positive. Spans with a positive prediction will be saved on the Doc.
+            Defaults to 0.5.
+        max_positive (Optional[int]): Maximum number of labels to consider
+            positive per span. Defaults to None, indicating no limit.
+        scorer (Optional[Callable]): The scoring method. Defaults to
+            Scorer.score_spans for the Doc.spans[spans_key] with overlapping
+            spans allowed.
+
+        DOCS: https://spacy.io/api/spancategorizer#init
+        """
+        self.cfg = {
+            "labels": [],
+            "spans_key": spans_key,
+            "threshold": threshold,
+            "max_positive": max_positive,
+        }
+        self.vocab = vocab
+        self.suggester = suggester
+        self.model = model
+        self.name = name
+        self.scorer = scorer
+
+    @property
+    def key(self) -> str:
+        """Key of the doc.spans dict to save the spans under. During
+        initialization and training, the component will look for spans on the
+        reference document under the same key.
+        """
+        return str(self.cfg["spans_key"])
+
+    def add_label(self, label: str) -> int:
+        """Add a new label to the pipe.
+
+        label (str): The label to add.
+        RETURNS (int): 0 if label is already present, otherwise 1.
+
+        DOCS: https://spacy.io/api/spancategorizer#add_label
+        """
+        if not isinstance(label, str):
+            raise ValueError(Errors.E187)
+        if label in self.labels:
+            return 0
+        self._allow_extra_label()
+        self.cfg["labels"].append(label)  # type: ignore
+        self.vocab.strings.add(label)
+        return 1
+
+    @property
+    def labels(self) -> Tuple[str]:
+        """RETURNS (Tuple[str]): The labels currently added to the component.
+
+        DOCS: https://spacy.io/api/spancategorizer#labels
+        """
+        return tuple(self.cfg["labels"])  # type: ignore
+
+    @property
+    def label_data(self) -> List[str]:
+        """RETURNS (List[str]): Information about the component's labels.
+
+        DOCS: https://spacy.io/api/spancategorizer#label_data
+        """
+        return list(self.labels)
+
+    def predict(self, docs: Iterable[Doc]):
+        """Apply the pipeline's model to a batch of docs, without modifying them.
+
+        docs (Iterable[Doc]): The documents to predict.
+        RETURNS: The models prediction for each document.
+
+        DOCS: https://spacy.io/api/spancategorizer#predict
+        """
+        indices = self.suggester(docs, ops=self.model.ops)
+        scores = self.model.predict((docs, indices))  # type: ignore
+        return indices, scores
+
+    def set_annotations(self, docs: Iterable[Doc], indices_scores) -> None:
+        """Modify a batch of Doc objects, using pre-computed scores.
+
+        docs (Iterable[Doc]): The documents to modify.
+        scores: The scores to set, produced by SpanCategorizer.predict.
+
+        DOCS: https://spacy.io/api/spancategorizer#set_annotations
+        """
+        labels = self.labels
+        indices, scores = indices_scores
+        offset = 0
+        for i, doc in enumerate(docs):
+            indices_i = indices[i].dataXd
+            doc.spans[self.key] = self._make_span_group(
+                doc, indices_i, scores[offset : offset + indices.lengths[i]], labels  # type: ignore[arg-type]
+            )
+            offset += indices.lengths[i]
+
+    def update(
+        self,
+        examples: Iterable[Example],
+        *,
+        drop: float = 0.0,
+        sgd: Optional[Optimizer] = None,
+        losses: Optional[Dict[str, float]] = None,
+    ) -> Dict[str, float]:
+        """Learn from a batch of documents and gold-standard information,
+        updating the pipe's model. Delegates to predict and get_loss.
+
+        examples (Iterable[Example]): A batch of Example objects.
+        drop (float): The dropout rate.
+        sgd (thinc.api.Optimizer): The optimizer.
+        losses (Dict[str, float]): Optional record of the loss during training.
+            Updated using the component name as the key.
+        RETURNS (Dict[str, float]): The updated losses dictionary.
+
+        DOCS: https://spacy.io/api/spancategorizer#update
+        """
+        if losses is None:
+            losses = {}
+        losses.setdefault(self.name, 0.0)
+        validate_examples(examples, "SpanCategorizer.update")
+        self._validate_categories(examples)
+        if not any(len(eg.predicted) if eg.predicted else 0 for eg in examples):
+            # Handle cases where there are no tokens in any docs.
+            return losses
+        docs = [eg.predicted for eg in examples]
+        spans = self.suggester(docs, ops=self.model.ops)
+        if spans.lengths.sum() == 0:
+            return losses
+        set_dropout_rate(self.model, drop)
+        scores, backprop_scores = self.model.begin_update((docs, spans))
+        loss, d_scores = self.get_loss(examples, (spans, scores))
+        backprop_scores(d_scores)  # type: ignore
+        if sgd is not None:
+            self.finish_update(sgd)
+        losses[self.name] += loss
+        return losses
+
+    def get_loss(
+        self, examples: Iterable[Example], spans_scores: Tuple[Ragged, Floats2d]
+    ) -> Tuple[float, float]:
+        """Find the loss and gradient of loss for the batch of documents and
+        their predicted scores.
+
+        examples (Iterable[Examples]): The batch of examples.
+        spans_scores: Scores representing the model's predictions.
+        RETURNS (Tuple[float, float]): The loss and the gradient.
+
+        DOCS: https://spacy.io/api/spancategorizer#get_loss
+        """
+        spans, scores = spans_scores
+        spans = Ragged(
+            self.model.ops.to_numpy(spans.data), self.model.ops.to_numpy(spans.lengths)
+        )
+        label_map = {label: i for i, label in enumerate(self.labels)}
+        target = numpy.zeros(scores.shape, dtype=scores.dtype)
+        offset = 0
+        for i, eg in enumerate(examples):
+            # Map (start, end) offset of spans to the row in the d_scores array,
+            # so that we can adjust the gradient for predictions that were
+            # in the gold standard.
+            spans_index = {}
+            spans_i = spans[i].dataXd
+            for j in range(spans.lengths[i]):
+                start = int(spans_i[j, 0])  # type: ignore
+                end = int(spans_i[j, 1])  # type: ignore
+                spans_index[(start, end)] = offset + j
+            for gold_span in self._get_aligned_spans(eg):
+                key = (gold_span.start, gold_span.end)
+                if key in spans_index:
+                    row = spans_index[key]
+                    k = label_map[gold_span.label_]
+                    target[row, k] = 1.0
+            # The target is a flat array for all docs. Track the position
+            # we're at within the flat array.
+            offset += spans.lengths[i]
+        target = self.model.ops.asarray(target, dtype="f")  # type: ignore
+        # The target will have the values 0 (for untrue predictions) or 1
+        # (for true predictions).
+        # The scores should be in the range [0, 1].
+        # If the prediction is 0.9 and it's true, the gradient
+        # will be -0.1 (0.9 - 1.0).
+        # If the prediction is 0.9 and it's false, the gradient will be
+        # 0.9 (0.9 - 0.0)
+        d_scores = scores - target
+        loss = float((d_scores ** 2).sum())
+        return loss, d_scores
+
+    def initialize(
+        self,
+        get_examples: Callable[[], Iterable[Example]],
+        *,
+        nlp: Optional[Language] = None,
+        labels: Optional[List[str]] = None,
+    ) -> None:
+        """Initialize the pipe for training, using a representative set
+        of data examples.
+
+        get_examples (Callable[[], Iterable[Example]]): Function that
+            returns a representative sample of gold-standard Example objects.
+        nlp (Optional[Language]): The current nlp object the component is part of.
+        labels (Optional[List[str]]): The labels to add to the component, typically generated by the
+            `init labels` command. If no labels are provided, the get_examples
+            callback is used to extract the labels from the data.
+
+        DOCS: https://spacy.io/api/spancategorizer#initialize
+        """
+        subbatch: List[Example] = []
+        if labels is not None:
+            for label in labels:
+                self.add_label(label)
+        for eg in get_examples():
+            if labels is None:
+                for span in eg.reference.spans.get(self.key, []):
+                    self.add_label(span.label_)
+            if len(subbatch) < 10:
+                subbatch.append(eg)
+        self._require_labels()
+        if subbatch:
+            docs = [eg.x for eg in subbatch]
+            spans = self.suggester(docs)
+            Y = self.model.ops.alloc2f(spans.dataXd.shape[0], len(self.labels))
+            self.model.initialize(X=(docs, spans), Y=Y)
+        else:
+            self.model.initialize()
+
+    def _validate_categories(self, examples: Iterable[Example]):
+        # TODO
+        pass
+
+    def _get_aligned_spans(self, eg: Example):
+        return eg.get_aligned_spans_y2x(
+            eg.reference.spans.get(self.key, []), allow_overlap=True
+        )
+
+    def _make_span_group(
+        self, doc: Doc, indices: Ints2d, scores: Floats2d, labels: List[str]
+    ) -> SpanGroup:
+        spans = SpanGroup(doc, name=self.key)
+        max_positive = self.cfg["max_positive"]
+        threshold = self.cfg["threshold"]
+
+        keeps = scores >= threshold
+        ranked = (scores * -1).argsort()  # type: ignore
+        if max_positive is not None:
+            assert isinstance(max_positive, int)
+            span_filter = ranked[:, max_positive:]
+            for i, row in enumerate(span_filter):
+                keeps[i, row] = False
+        spans.attrs["scores"] = scores[keeps].flatten()
+
+        indices = self.model.ops.to_numpy(indices)
+        keeps = self.model.ops.to_numpy(keeps)
+
+        for i in range(indices.shape[0]):
+            start = indices[i, 0]
+            end = indices[i, 1]
+
+            for j, keep in enumerate(keeps[i]):
+                if keep:
+                    spans.append(Span(doc, start, end, label=labels[j]))
+
+        return spans

cgpenv/Lib/site-packages/spacy/pipeline/tagger.pyx

+# cython: infer_types=True, profile=True, binding=True
+from typing import Callable, Optional
+import numpy
+import srsly
+from thinc.api import Model, set_dropout_rate, SequenceCategoricalCrossentropy, Config
+from thinc.types import Floats2d
+import warnings
+from itertools import islice
+
+from ..tokens.doc cimport Doc
+from ..morphology cimport Morphology
+from ..vocab cimport Vocab
+
+from .trainable_pipe import TrainablePipe
+from .pipe import deserialize_config
+from ..language import Language
+from ..attrs import POS, ID
+from ..parts_of_speech import X
+from ..errors import Errors, Warnings
+from ..scorer import Scorer
+from ..training import validate_examples, validate_get_examples
+from ..util import registry
+from .. import util
+
+# See #9050
+BACKWARD_OVERWRITE = False
+
+default_model_config = """
+[model]
+@architectures = "spacy.Tagger.v1"
+
+[model.tok2vec]
+@architectures = "spacy.HashEmbedCNN.v2"
+pretrained_vectors = null
+width = 96
+depth = 4
+embed_size = 2000
+window_size = 1
+maxout_pieces = 3
+subword_features = true
+"""
+DEFAULT_TAGGER_MODEL = Config().from_str(default_model_config)["model"]
+
+
+@Language.factory(
+    "tagger",
+    assigns=["token.tag"],
+    default_config={"model": DEFAULT_TAGGER_MODEL, "overwrite": False, "scorer": {"@scorers": "spacy.tagger_scorer.v1"}},
+    default_score_weights={"tag_acc": 1.0},
+)
+def make_tagger(
+    nlp: Language,
+    name: str,
+    model: Model,
+    overwrite: bool,
+    scorer: Optional[Callable],
+):
+    """Construct a part-of-speech tagger component.
+
+    model (Model[List[Doc], List[Floats2d]]): A model instance that predicts
+        the tag probabilities. The output vectors should match the number of tags
+        in size, and be normalized as probabilities (all scores between 0 and 1,
+        with the rows summing to 1).
+    """
+    return Tagger(nlp.vocab, model, name, overwrite=overwrite, scorer=scorer)
+
+
+def tagger_score(examples, **kwargs):
+    return Scorer.score_token_attr(examples, "tag", **kwargs)
+
+
+@registry.scorers("spacy.tagger_scorer.v1")
+def make_tagger_scorer():
+    return tagger_score
+
+
+class Tagger(TrainablePipe):
+    """Pipeline component for part-of-speech tagging.
+
+    DOCS: https://spacy.io/api/tagger
+    """
+    def __init__(
+        self,
+        vocab,
+        model,
+        name="tagger",
+        *,
+        overwrite=BACKWARD_OVERWRITE,
+        scorer=tagger_score,
+    ):
+        """Initialize a part-of-speech tagger.
+
+        vocab (Vocab): The shared vocabulary.
+        model (thinc.api.Model): The Thinc Model powering the pipeline component.
+        name (str): The component instance name, used to add entries to the
+            losses during training.
+        scorer (Optional[Callable]): The scoring method. Defaults to
+            Scorer.score_token_attr for the attribute "tag".
+
+        DOCS: https://spacy.io/api/tagger#init
+        """
+        self.vocab = vocab
+        self.model = model
+        self.name = name
+        self._rehearsal_model = None
+        cfg = {"labels": [], "overwrite": overwrite}
+        self.cfg = dict(sorted(cfg.items()))
+        self.scorer = scorer
+
+    @property
+    def labels(self):
+        """The labels currently added to the component. Note that even for a
+        blank component, this will always include the built-in coarse-grained
+        part-of-speech tags by default.
+
+        RETURNS (Tuple[str]): The labels.
+
+        DOCS: https://spacy.io/api/tagger#labels
+        """
+        return tuple(self.cfg["labels"])
+
+    @property
+    def label_data(self):
+        """Data about the labels currently added to the component."""
+        return tuple(self.cfg["labels"])
+
+    def predict(self, docs):
+        """Apply the pipeline's model to a batch of docs, without modifying them.
+
+        docs (Iterable[Doc]): The documents to predict.
+        RETURNS: The models prediction for each document.
+
+        DOCS: https://spacy.io/api/tagger#predict
+        """
+        if not any(len(doc) for doc in docs):
+            # Handle cases where there are no tokens in any docs.
+            n_labels = len(self.labels)
+            guesses = [self.model.ops.alloc((0, n_labels)) for doc in docs]
+            assert len(guesses) == len(docs)
+            return guesses
+        scores = self.model.predict(docs)
+        assert len(scores) == len(docs), (len(scores), len(docs))
+        guesses = self._scores2guesses(scores)
+        assert len(guesses) == len(docs)
+        return guesses
+
+    def _scores2guesses(self, scores):
+        guesses = []
+        for doc_scores in scores:
+            doc_guesses = doc_scores.argmax(axis=1)
+            if not isinstance(doc_guesses, numpy.ndarray):
+                doc_guesses = doc_guesses.get()
+            guesses.append(doc_guesses)
+        return guesses
+
+    def set_annotations(self, docs, batch_tag_ids):
+        """Modify a batch of documents, using pre-computed scores.
+
+        docs (Iterable[Doc]): The documents to modify.
+        batch_tag_ids: The IDs to set, produced by Tagger.predict.
+
+        DOCS: https://spacy.io/api/tagger#set_annotations
+        """
+        if isinstance(docs, Doc):
+            docs = [docs]
+        cdef Doc doc
+        cdef Vocab vocab = self.vocab
+        cdef bint overwrite = self.cfg["overwrite"]
+        for i, doc in enumerate(docs):
+            doc_tag_ids = batch_tag_ids[i]
+            if hasattr(doc_tag_ids, "get"):
+                doc_tag_ids = doc_tag_ids.get()
+            for j, tag_id in enumerate(doc_tag_ids):
+                if doc.c[j].tag == 0 or overwrite:
+                    doc.c[j].tag = self.vocab.strings[self.labels[tag_id]]
+
+    def update(self, examples, *, drop=0., sgd=None, losses=None):
+        """Learn from a batch of documents and gold-standard information,
+        updating the pipe's model. Delegates to predict and get_loss.
+
+        examples (Iterable[Example]): A batch of Example objects.
+        drop (float): The dropout rate.
+        sgd (thinc.api.Optimizer): The optimizer.
+        losses (Dict[str, float]): Optional record of the loss during training.
+            Updated using the component name as the key.
+        RETURNS (Dict[str, float]): The updated losses dictionary.
+
+        DOCS: https://spacy.io/api/tagger#update
+        """
+        if losses is None:
+            losses = {}
+        losses.setdefault(self.name, 0.0)
+        validate_examples(examples, "Tagger.update")
+        if not any(len(eg.predicted) if eg.predicted else 0 for eg in examples):
+            # Handle cases where there are no tokens in any docs.
+            return losses
+        set_dropout_rate(self.model, drop)
+        tag_scores, bp_tag_scores = self.model.begin_update([eg.predicted for eg in examples])
+        for sc in tag_scores:
+            if self.model.ops.xp.isnan(sc.sum()):
+                raise ValueError(Errors.E940)
+        loss, d_tag_scores = self.get_loss(examples, tag_scores)
+        bp_tag_scores(d_tag_scores)
+        if sgd not in (None, False):
+            self.finish_update(sgd)
+
+        losses[self.name] += loss
+        return losses
+
+    def rehearse(self, examples, *, drop=0., sgd=None, losses=None):
+        """Perform a "rehearsal" update from a batch of data. Rehearsal updates
+        teach the current model to make predictions similar to an initial model,
+        to try to address the "catastrophic forgetting" problem. This feature is
+        experimental.
+
+        examples (Iterable[Example]): A batch of Example objects.
+        drop (float): The dropout rate.
+        sgd (thinc.api.Optimizer): The optimizer.
+        losses (Dict[str, float]): Optional record of the loss during training.
+            Updated using the component name as the key.
+        RETURNS (Dict[str, float]): The updated losses dictionary.
+
+        DOCS: https://spacy.io/api/tagger#rehearse
+        """
+        if losses is None:
+            losses = {}
+        losses.setdefault(self.name, 0.0)
+        validate_examples(examples, "Tagger.rehearse")
+        docs = [eg.predicted for eg in examples]
+        if self._rehearsal_model is None:
+            return losses
+        if not any(len(doc) for doc in docs):
+            # Handle cases where there are no tokens in any docs.
+            return losses
+        set_dropout_rate(self.model, drop)
+        guesses, backprop = self.model.begin_update(docs)
+        target = self._rehearsal_model(examples)
+        gradient = guesses - target
+        backprop(gradient)
+        self.finish_update(sgd)
+        losses[self.name] += (gradient**2).sum()
+        return losses
+
+    def get_loss(self, examples, scores):
+        """Find the loss and gradient of loss for the batch of documents and
+        their predicted scores.
+
+        examples (Iterable[Examples]): The batch of examples.
+        scores: Scores representing the model's predictions.
+        RETURNS (Tuple[float, float]): The loss and the gradient.
+
+        DOCS: https://spacy.io/api/tagger#get_loss
+        """
+        validate_examples(examples, "Tagger.get_loss")
+        loss_func = SequenceCategoricalCrossentropy(names=self.labels, normalize=False, neg_prefix="!")
+        # Convert empty tag "" to missing value None so that both misaligned
+        # tokens and tokens with missing annotation have the default missing
+        # value None.
+        truths = []
+        for eg in examples:
+            eg_truths = [tag if tag is not "" else None for tag in eg.get_aligned("TAG", as_string=True)]
+            truths.append(eg_truths)
+        d_scores, loss = loss_func(scores, truths)
+        if self.model.ops.xp.isnan(loss):
+            raise ValueError(Errors.E910.format(name=self.name))
+        return float(loss), d_scores
+
+    def initialize(self, get_examples, *, nlp=None, labels=None):
+        """Initialize the pipe for training, using a representative set
+        of data examples.
+
+        get_examples (Callable[[], Iterable[Example]]): Function that
+            returns a representative sample of gold-standard Example objects..
+        nlp (Language): The current nlp object the component is part of.
+        labels: The labels to add to the component, typically generated by the
+            `init labels` command. If no labels are provided, the get_examples
+            callback is used to extract the labels from the data.
+
+        DOCS: https://spacy.io/api/tagger#initialize
+        """
+        validate_get_examples(get_examples, "Tagger.initialize")
+        util.check_lexeme_norms(self.vocab, "tagger")
+        if labels is not None:
+            for tag in labels:
+                self.add_label(tag)
+        else:
+            tags = set()
+            for example in get_examples():
+                for token in example.y:
+                    if token.tag_:
+                        tags.add(token.tag_)
+            for tag in sorted(tags):
+                self.add_label(tag)
+        doc_sample = []
+        label_sample = []
+        for example in islice(get_examples(), 10):
+            doc_sample.append(example.x)
+            gold_tags = example.get_aligned("TAG", as_string=True)
+            gold_array = [[1.0 if tag == gold_tag else 0.0 for tag in self.labels] for gold_tag in gold_tags]
+            label_sample.append(self.model.ops.asarray(gold_array, dtype="float32"))
+        self._require_labels()
+        assert len(doc_sample) > 0, Errors.E923.format(name=self.name)
+        assert len(label_sample) > 0, Errors.E923.format(name=self.name)
+        self.model.initialize(X=doc_sample, Y=label_sample)
+
+    def add_label(self, label):
+        """Add a new label to the pipe.
+
+        label (str): The label to add.
+        RETURNS (int): 0 if label is already present, otherwise 1.
+
+        DOCS: https://spacy.io/api/tagger#add_label
+        """
+        if not isinstance(label, str):
+            raise ValueError(Errors.E187)
+        if label in self.labels:
+            return 0
+        self._allow_extra_label()
+        self.cfg["labels"].append(label)
+        self.vocab.strings.add(label)
+        return 1

cgpenv/Lib/site-packages/spacy/pipeline/textcat.py

+from itertools import islice
+from typing import Iterable, Tuple, Optional, Dict, List, Callable, Any
+from thinc.api import get_array_module, Model, Optimizer, set_dropout_rate, Config
+from thinc.types import Floats2d
+import numpy
+
+from .trainable_pipe import TrainablePipe
+from ..language import Language
+from ..training import Example, validate_examples, validate_get_examples
+from ..errors import Errors
+from ..scorer import Scorer
+from ..tokens import Doc
+from ..util import registry
+from ..vocab import Vocab
+
+
+single_label_default_config = """
+[model]
+@architectures = "spacy.TextCatEnsemble.v2"
+
+[model.tok2vec]
+@architectures = "spacy.Tok2Vec.v2"
+
+[model.tok2vec.embed]
+@architectures = "spacy.MultiHashEmbed.v2"
+width = 64
+rows = [2000, 2000, 1000, 1000, 1000, 1000]
+attrs = ["ORTH", "LOWER", "PREFIX", "SUFFIX", "SHAPE", "ID"]
+include_static_vectors = false
+
+[model.tok2vec.encode]
+@architectures = "spacy.MaxoutWindowEncoder.v2"
+width = ${model.tok2vec.embed.width}
+window_size = 1
+maxout_pieces = 3
+depth = 2
+
+[model.linear_model]
+@architectures = "spacy.TextCatBOW.v2"
+exclusive_classes = true
+ngram_size = 1
+no_output_layer = false
+"""
+DEFAULT_SINGLE_TEXTCAT_MODEL = Config().from_str(single_label_default_config)["model"]
+
+single_label_bow_config = """
+[model]
+@architectures = "spacy.TextCatBOW.v2"
+exclusive_classes = true
+ngram_size = 1
+no_output_layer = false
+"""
+
+single_label_cnn_config = """
+[model]
+@architectures = "spacy.TextCatCNN.v2"
+exclusive_classes = true
+
+[model.tok2vec]
+@architectures = "spacy.HashEmbedCNN.v2"
+pretrained_vectors = null
+width = 96
+depth = 4
+embed_size = 2000
+window_size = 1
+maxout_pieces = 3
+subword_features = true
+"""
+
+
+@Language.factory(
+    "textcat",
+    assigns=["doc.cats"],
+    default_config={
+        "threshold": 0.5,
+        "model": DEFAULT_SINGLE_TEXTCAT_MODEL,
+        "scorer": {"@scorers": "spacy.textcat_scorer.v1"},
+    },
+    default_score_weights={
+        "cats_score": 1.0,
+        "cats_score_desc": None,
+        "cats_micro_p": None,
+        "cats_micro_r": None,
+        "cats_micro_f": None,
+        "cats_macro_p": None,
+        "cats_macro_r": None,
+        "cats_macro_f": None,
+        "cats_macro_auc": None,
+        "cats_f_per_type": None,
+        "cats_macro_auc_per_type": None,
+    },
+)
+def make_textcat(
+    nlp: Language,
+    name: str,
+    model: Model[List[Doc], List[Floats2d]],
+    threshold: float,
+    scorer: Optional[Callable],
+) -> "TextCategorizer":
+    """Create a TextCategorizer component. The text categorizer predicts categories
+    over a whole document. It can learn one or more labels, and the labels are considered
+    to be mutually exclusive (i.e. one true label per doc).
+
+    model (Model[List[Doc], List[Floats2d]]): A model instance that predicts
+        scores for each category.
+    threshold (float): Cutoff to consider a prediction "positive".
+    scorer (Optional[Callable]): The scoring method.
+    """
+    return TextCategorizer(nlp.vocab, model, name, threshold=threshold, scorer=scorer)
+
+
+def textcat_score(examples: Iterable[Example], **kwargs) -> Dict[str, Any]:
+    return Scorer.score_cats(
+        examples,
+        "cats",
+        multi_label=False,
+        **kwargs,
+    )
+
+
+@registry.scorers("spacy.textcat_scorer.v1")
+def make_textcat_scorer():
+    return textcat_score
+
+
+class TextCategorizer(TrainablePipe):
+    """Pipeline component for single-label text classification.
+
+    DOCS: https://spacy.io/api/textcategorizer
+    """
+
+    def __init__(
+        self,
+        vocab: Vocab,
+        model: Model,
+        name: str = "textcat",
+        *,
+        threshold: float,
+        scorer: Optional[Callable] = textcat_score,
+    ) -> None:
+        """Initialize a text categorizer for single-label classification.
+
+        vocab (Vocab): The shared vocabulary.
+        model (thinc.api.Model): The Thinc Model powering the pipeline component.
+        name (str): The component instance name, used to add entries to the
+            losses during training.
+        threshold (float): Cutoff to consider a prediction "positive".
+        scorer (Optional[Callable]): The scoring method. Defaults to
+                Scorer.score_cats for the attribute "cats".
+
+        DOCS: https://spacy.io/api/textcategorizer#init
+        """
+        self.vocab = vocab
+        self.model = model
+        self.name = name
+        self._rehearsal_model = None
+        cfg = {"labels": [], "threshold": threshold, "positive_label": None}
+        self.cfg = dict(cfg)
+        self.scorer = scorer
+
+    @property
+    def labels(self) -> Tuple[str]:
+        """RETURNS (Tuple[str]): The labels currently added to the component.
+
+        DOCS: https://spacy.io/api/textcategorizer#labels
+        """
+        return tuple(self.cfg["labels"])  # type: ignore[arg-type, return-value]
+
+    @property
+    def label_data(self) -> List[str]:
+        """RETURNS (List[str]): Information about the component's labels.
+
+        DOCS: https://spacy.io/api/textcategorizer#label_data
+        """
+        return self.labels  # type: ignore[return-value]
+
+    def predict(self, docs: Iterable[Doc]):
+        """Apply the pipeline's model to a batch of docs, without modifying them.
+
+        docs (Iterable[Doc]): The documents to predict.
+        RETURNS: The models prediction for each document.
+
+        DOCS: https://spacy.io/api/textcategorizer#predict
+        """
+        if not any(len(doc) for doc in docs):
+            # Handle cases where there are no tokens in any docs.
+            tensors = [doc.tensor for doc in docs]
+            xp = get_array_module(tensors)
+            scores = xp.zeros((len(list(docs)), len(self.labels)))
+            return scores
+        scores = self.model.predict(docs)
+        scores = self.model.ops.asarray(scores)
+        return scores
+
+    def set_annotations(self, docs: Iterable[Doc], scores) -> None:
+        """Modify a batch of Doc objects, using pre-computed scores.
+
+        docs (Iterable[Doc]): The documents to modify.
+        scores: The scores to set, produced by TextCategorizer.predict.
+
+        DOCS: https://spacy.io/api/textcategorizer#set_annotations
+        """
+        for i, doc in enumerate(docs):
+            for j, label in enumerate(self.labels):
+                doc.cats[label] = float(scores[i, j])
+
+    def update(
+        self,
+        examples: Iterable[Example],
+        *,
+        drop: float = 0.0,
+        sgd: Optional[Optimizer] = None,
+        losses: Optional[Dict[str, float]] = None,
+    ) -> Dict[str, float]:
+        """Learn from a batch of documents and gold-standard information,
+        updating the pipe's model. Delegates to predict and get_loss.
+
+        examples (Iterable[Example]): A batch of Example objects.
+        drop (float): The dropout rate.
+        sgd (thinc.api.Optimizer): The optimizer.
+        losses (Dict[str, float]): Optional record of the loss during training.
+            Updated using the component name as the key.
+        RETURNS (Dict[str, float]): The updated losses dictionary.
+
+        DOCS: https://spacy.io/api/textcategorizer#update
+        """
+        if losses is None:
+            losses = {}
+        losses.setdefault(self.name, 0.0)
+        validate_examples(examples, "TextCategorizer.update")
+        self._validate_categories(examples)
+        if not any(len(eg.predicted) if eg.predicted else 0 for eg in examples):
+            # Handle cases where there are no tokens in any docs.
+            return losses
+        set_dropout_rate(self.model, drop)
+        scores, bp_scores = self.model.begin_update([eg.predicted for eg in examples])
+        loss, d_scores = self.get_loss(examples, scores)
+        bp_scores(d_scores)
+        if sgd is not None:
+            self.finish_update(sgd)
+        losses[self.name] += loss
+        return losses
+
+    def rehearse(
+        self,
+        examples: Iterable[Example],
+        *,
+        drop: float = 0.0,
+        sgd: Optional[Optimizer] = None,
+        losses: Optional[Dict[str, float]] = None,
+    ) -> Dict[str, float]:
+        """Perform a "rehearsal" update from a batch of data. Rehearsal updates
+        teach the current model to make predictions similar to an initial model,
+        to try to address the "catastrophic forgetting" problem. This feature is
+        experimental.
+
+        examples (Iterable[Example]): A batch of Example objects.
+        drop (float): The dropout rate.
+        sgd (thinc.api.Optimizer): The optimizer.
+        losses (Dict[str, float]): Optional record of the loss during training.
+            Updated using the component name as the key.
+        RETURNS (Dict[str, float]): The updated losses dictionary.
+
+        DOCS: https://spacy.io/api/textcategorizer#rehearse
+        """
+        if losses is None:
+            losses = {}
+        losses.setdefault(self.name, 0.0)
+        if self._rehearsal_model is None:
+            return losses
+        validate_examples(examples, "TextCategorizer.rehearse")
+        self._validate_categories(examples)
+        docs = [eg.predicted for eg in examples]
+        if not any(len(doc) for doc in docs):
+            # Handle cases where there are no tokens in any docs.
+            return losses
+        set_dropout_rate(self.model, drop)
+        scores, bp_scores = self.model.begin_update(docs)
+        target = self._rehearsal_model(examples)
+        gradient = scores - target
+        bp_scores(gradient)
+        if sgd is not None:
+            self.finish_update(sgd)
+        losses[self.name] += (gradient ** 2).sum()
+        return losses
+
+    def _examples_to_truth(
+        self, examples: Iterable[Example]
+    ) -> Tuple[numpy.ndarray, numpy.ndarray]:
+        nr_examples = len(list(examples))
+        truths = numpy.zeros((nr_examples, len(self.labels)), dtype="f")
+        not_missing = numpy.ones((nr_examples, len(self.labels)), dtype="f")
+        for i, eg in enumerate(examples):
+            for j, label in enumerate(self.labels):
+                if label in eg.reference.cats:
+                    truths[i, j] = eg.reference.cats[label]
+                else:
+                    not_missing[i, j] = 0.0
+        truths = self.model.ops.asarray(truths)  # type: ignore
+        return truths, not_missing  # type: ignore
+
+    def get_loss(self, examples: Iterable[Example], scores) -> Tuple[float, float]:
+        """Find the loss and gradient of loss for the batch of documents and
+        their predicted scores.
+
+        examples (Iterable[Examples]): The batch of examples.
+        scores: Scores representing the model's predictions.
+        RETURNS (Tuple[float, float]): The loss and the gradient.
+
+        DOCS: https://spacy.io/api/textcategorizer#get_loss
+        """
+        validate_examples(examples, "TextCategorizer.get_loss")
+        self._validate_categories(examples)
+        truths, not_missing = self._examples_to_truth(examples)
+        not_missing = self.model.ops.asarray(not_missing)  # type: ignore
+        d_scores = (scores - truths) / scores.shape[0]
+        d_scores *= not_missing
+        mean_square_error = (d_scores ** 2).sum(axis=1).mean()
+        return float(mean_square_error), d_scores
+
+    def add_label(self, label: str) -> int:
+        """Add a new label to the pipe.
+
+        label (str): The label to add.
+        RETURNS (int): 0 if label is already present, otherwise 1.
+
+        DOCS: https://spacy.io/api/textcategorizer#add_label
+        """
+        if not isinstance(label, str):
+            raise ValueError(Errors.E187)
+        if label in self.labels:
+            return 0
+        self._allow_extra_label()
+        self.cfg["labels"].append(label)  # type: ignore[attr-defined]
+        if self.model and "resize_output" in self.model.attrs:
+            self.model = self.model.attrs["resize_output"](self.model, len(self.labels))
+        self.vocab.strings.add(label)
+        return 1
+
+    def initialize(
+        self,
+        get_examples: Callable[[], Iterable[Example]],
+        *,
+        nlp: Optional[Language] = None,
+        labels: Optional[Iterable[str]] = None,
+        positive_label: Optional[str] = None,
+    ) -> None:
+        """Initialize the pipe for training, using a representative set
+        of data examples.
+
+        get_examples (Callable[[], Iterable[Example]]): Function that
+            returns a representative sample of gold-standard Example objects.
+        nlp (Language): The current nlp object the component is part of.
+        labels (Optional[Iterable[str]]): The labels to add to the component, typically generated by the
+            `init labels` command. If no labels are provided, the get_examples
+            callback is used to extract the labels from the data.
+        positive_label (Optional[str]): The positive label for a binary task with exclusive classes,
+            `None` otherwise and by default.
+
+        DOCS: https://spacy.io/api/textcategorizer#initialize
+        """
+        validate_get_examples(get_examples, "TextCategorizer.initialize")
+        self._validate_categories(get_examples())
+        if labels is None:
+            for example in get_examples():
+                for cat in example.y.cats:
+                    self.add_label(cat)
+        else:
+            for label in labels:
+                self.add_label(label)
+        if len(self.labels) < 2:
+            raise ValueError(Errors.E867)
+        if positive_label is not None:
+            if positive_label not in self.labels:
+                err = Errors.E920.format(pos_label=positive_label, labels=self.labels)
+                raise ValueError(err)
+            if len(self.labels) != 2:
+                err = Errors.E919.format(pos_label=positive_label, labels=self.labels)
+                raise ValueError(err)
+        self.cfg["positive_label"] = positive_label
+        subbatch = list(islice(get_examples(), 10))
+        doc_sample = [eg.reference for eg in subbatch]
+        label_sample, _ = self._examples_to_truth(subbatch)
+        self._require_labels()
+        assert len(doc_sample) > 0, Errors.E923.format(name=self.name)
+        assert len(label_sample) > 0, Errors.E923.format(name=self.name)
+        self.model.initialize(X=doc_sample, Y=label_sample)
+
+    def _validate_categories(self, examples: Iterable[Example]):
+        """Check whether the provided examples all have single-label cats annotations."""
+        for ex in examples:
+            if list(ex.reference.cats.values()).count(1.0) > 1:
+                raise ValueError(Errors.E895.format(value=ex.reference.cats))

cgpenv/Lib/site-packages/spacy/pipeline/textcat_multilabel.py

+from itertools import islice
+from typing import Iterable, Optional, Dict, List, Callable, Any
+
+from thinc.api import Model, Config
+from thinc.types import Floats2d
+
+from ..language import Language
+from ..training import Example, validate_get_examples
+from ..errors import Errors
+from ..scorer import Scorer
+from ..tokens import Doc
+from ..util import registry
+from ..vocab import Vocab
+from .textcat import TextCategorizer
+
+
+multi_label_default_config = """
+[model]
+@architectures = "spacy.TextCatEnsemble.v2"
+
+[model.tok2vec]
+@architectures = "spacy.Tok2Vec.v1"
+
+[model.tok2vec.embed]
+@architectures = "spacy.MultiHashEmbed.v2"
+width = 64
+rows = [2000, 2000, 1000, 1000, 1000, 1000]
+attrs = ["ORTH", "LOWER", "PREFIX", "SUFFIX", "SHAPE", "ID"]
+include_static_vectors = false
+
+[model.tok2vec.encode]
+@architectures = "spacy.MaxoutWindowEncoder.v1"
+width = ${model.tok2vec.embed.width}
+window_size = 1
+maxout_pieces = 3
+depth = 2
+
+[model.linear_model]
+@architectures = "spacy.TextCatBOW.v2"
+exclusive_classes = false
+ngram_size = 1
+no_output_layer = false
+"""
+DEFAULT_MULTI_TEXTCAT_MODEL = Config().from_str(multi_label_default_config)["model"]
+
+multi_label_bow_config = """
+[model]
+@architectures = "spacy.TextCatBOW.v2"
+exclusive_classes = false
+ngram_size = 1
+no_output_layer = false
+"""
+
+multi_label_cnn_config = """
+[model]
+@architectures = "spacy.TextCatCNN.v2"
+exclusive_classes = false
+
+[model.tok2vec]
+@architectures = "spacy.HashEmbedCNN.v2"
+pretrained_vectors = null
+width = 96
+depth = 4
+embed_size = 2000
+window_size = 1
+maxout_pieces = 3
+subword_features = true
+"""
+
+
+@Language.factory(
+    "textcat_multilabel",
+    assigns=["doc.cats"],
+    default_config={
+        "threshold": 0.5,
+        "model": DEFAULT_MULTI_TEXTCAT_MODEL,
+        "scorer": {"@scorers": "spacy.textcat_multilabel_scorer.v1"},
+    },
+    default_score_weights={
+        "cats_score": 1.0,
+        "cats_score_desc": None,
+        "cats_micro_p": None,
+        "cats_micro_r": None,
+        "cats_micro_f": None,
+        "cats_macro_p": None,
+        "cats_macro_r": None,
+        "cats_macro_f": None,
+        "cats_macro_auc": None,
+        "cats_f_per_type": None,
+        "cats_macro_auc_per_type": None,
+    },
+)
+def make_multilabel_textcat(
+    nlp: Language,
+    name: str,
+    model: Model[List[Doc], List[Floats2d]],
+    threshold: float,
+    scorer: Optional[Callable],
+) -> "TextCategorizer":
+    """Create a TextCategorizer component. The text categorizer predicts categories
+    over a whole document. It can learn one or more labels, and the labels are considered
+    to be non-mutually exclusive, which means that there can be zero or more labels
+    per doc).
+
+    model (Model[List[Doc], List[Floats2d]]): A model instance that predicts
+        scores for each category.
+    threshold (float): Cutoff to consider a prediction "positive".
+    """
+    return MultiLabel_TextCategorizer(
+        nlp.vocab, model, name, threshold=threshold, scorer=scorer
+    )
+
+
+def textcat_multilabel_score(examples: Iterable[Example], **kwargs) -> Dict[str, Any]:
+    return Scorer.score_cats(
+        examples,
+        "cats",
+        multi_label=True,
+        **kwargs,
+    )
+
+
+@registry.scorers("spacy.textcat_multilabel_scorer.v1")
+def make_textcat_multilabel_scorer():
+    return textcat_multilabel_score
+
+
+class MultiLabel_TextCategorizer(TextCategorizer):
+    """Pipeline component for multi-label text classification.
+
+    DOCS: https://spacy.io/api/textcategorizer
+    """
+
+    def __init__(
+        self,
+        vocab: Vocab,
+        model: Model,
+        name: str = "textcat_multilabel",
+        *,
+        threshold: float,
+        scorer: Optional[Callable] = textcat_multilabel_score,
+    ) -> None:
+        """Initialize a text categorizer for multi-label classification.
+
+        vocab (Vocab): The shared vocabulary.
+        model (thinc.api.Model): The Thinc Model powering the pipeline component.
+        name (str): The component instance name, used to add entries to the
+            losses during training.
+        threshold (float): Cutoff to consider a prediction "positive".
+
+        DOCS: https://spacy.io/api/textcategorizer#init
+        """
+        self.vocab = vocab
+        self.model = model
+        self.name = name
+        self._rehearsal_model = None
+        cfg = {"labels": [], "threshold": threshold}
+        self.cfg = dict(cfg)
+        self.scorer = scorer
+
+    def initialize(  # type: ignore[override]
+        self,
+        get_examples: Callable[[], Iterable[Example]],
+        *,
+        nlp: Optional[Language] = None,
+        labels: Optional[Iterable[str]] = None,
+    ):
+        """Initialize the pipe for training, using a representative set
+        of data examples.
+
+        get_examples (Callable[[], Iterable[Example]]): Function that
+            returns a representative sample of gold-standard Example objects.
+        nlp (Language): The current nlp object the component is part of.
+        labels: The labels to add to the component, typically generated by the
+            `init labels` command. If no labels are provided, the get_examples
+            callback is used to extract the labels from the data.
+
+        DOCS: https://spacy.io/api/textcategorizer#initialize
+        """
+        validate_get_examples(get_examples, "MultiLabel_TextCategorizer.initialize")
+        if labels is None:
+            for example in get_examples():
+                for cat in example.y.cats:
+                    self.add_label(cat)
+        else:
+            for label in labels:
+                self.add_label(label)
+        subbatch = list(islice(get_examples(), 10))
+        doc_sample = [eg.reference for eg in subbatch]
+        label_sample, _ = self._examples_to_truth(subbatch)
+        self._require_labels()
+        assert len(doc_sample) > 0, Errors.E923.format(name=self.name)
+        assert len(label_sample) > 0, Errors.E923.format(name=self.name)
+        self.model.initialize(X=doc_sample, Y=label_sample)
+
+    def _validate_categories(self, examples: Iterable[Example]):
+        """This component allows any type of single- or multi-label annotations.
+        This method overwrites the more strict one from 'textcat'."""
+        pass

cgpenv/Lib/site-packages/spacy/pipeline/tok2vec.py

+from typing import Sequence, Iterable, Optional, Dict, Callable, List, Any
+from thinc.api import Model, set_dropout_rate, Optimizer, Config
+from itertools import islice
+
+from .trainable_pipe import TrainablePipe
+from ..training import Example, validate_examples, validate_get_examples
+from ..tokens import Doc
+from ..vocab import Vocab
+from ..language import Language
+from ..errors import Errors
+
+default_model_config = """
+[model]
+@architectures = "spacy.HashEmbedCNN.v2"
+pretrained_vectors = null
+width = 96
+depth = 4
+embed_size = 2000
+window_size = 1
+maxout_pieces = 3
+subword_features = true
+"""
+DEFAULT_TOK2VEC_MODEL = Config().from_str(default_model_config)["model"]
+
+
+@Language.factory(
+    "tok2vec", assigns=["doc.tensor"], default_config={"model": DEFAULT_TOK2VEC_MODEL}
+)
+def make_tok2vec(nlp: Language, name: str, model: Model) -> "Tok2Vec":
+    return Tok2Vec(nlp.vocab, model, name)
+
+
+class Tok2Vec(TrainablePipe):
+    """Apply a "token-to-vector" model and set its outputs in the doc.tensor
+    attribute. This is mostly useful to share a single subnetwork between multiple
+    components, e.g. to have one embedding and CNN network shared between a
+    parser, tagger and NER.
+
+    In order to use the `Tok2Vec` predictions, subsequent components should use
+    the `Tok2VecListener` layer as the tok2vec subnetwork of their model. This
+    layer will read data from the `doc.tensor` attribute during prediction.
+    During training, the `Tok2Vec` component will save its prediction and backprop
+    callback for each batch, so that the subsequent components can backpropagate
+    to the shared weights. This implementation is used because it allows us to
+    avoid relying on object identity within the models to achieve the parameter
+    sharing.
+    """
+
+    def __init__(self, vocab: Vocab, model: Model, name: str = "tok2vec") -> None:
+        """Initialize a tok2vec component.
+
+        vocab (Vocab): The shared vocabulary.
+        model (thinc.api.Model[List[Doc], List[Floats2d]]):
+            The Thinc Model powering the pipeline component. It should take
+            a list of Doc objects as input, and output a list of 2d float arrays.
+        name (str): The component instance name.
+
+        DOCS: https://spacy.io/api/tok2vec#init
+        """
+        self.vocab = vocab
+        self.model = model
+        self.name = name
+        self.listener_map: Dict[str, List["Tok2VecListener"]] = {}
+        self.cfg: Dict[str, Any] = {}
+
+    @property
+    def listeners(self) -> List["Tok2VecListener"]:
+        """RETURNS (List[Tok2VecListener]): The listener models listening to this
+        component. Usually internals.
+        """
+        return [m for c in self.listening_components for m in self.listener_map[c]]
+
+    @property
+    def listening_components(self) -> List[str]:
+        """RETURNS (List[str]): The downstream components listening to this
+        component. Usually internals.
+        """
+        return list(self.listener_map.keys())
+
+    def add_listener(self, listener: "Tok2VecListener", component_name: str) -> None:
+        """Add a listener for a downstream component. Usually internals."""
+        self.listener_map.setdefault(component_name, [])
+        if listener not in self.listener_map[component_name]:
+            self.listener_map[component_name].append(listener)
+
+    def remove_listener(self, listener: "Tok2VecListener", component_name: str) -> bool:
+        """Remove a listener for a downstream component. Usually internals."""
+        if component_name in self.listener_map:
+            if listener in self.listener_map[component_name]:
+                self.listener_map[component_name].remove(listener)
+                # If no listeners are left, remove entry
+                if not self.listener_map[component_name]:
+                    del self.listener_map[component_name]
+                return True
+        return False
+
+    def find_listeners(self, component) -> None:
+        """Walk over a model of a processing component, looking for layers that
+        are Tok2vecListener subclasses that have an upstream_name that matches
+        this component. Listeners can also set their upstream_name attribute to
+        the wildcard string '*' to match any `Tok2Vec`.
+
+        You're unlikely to ever need multiple `Tok2Vec` components, so it's
+        fine to leave your listeners upstream_name on '*'.
+        """
+        names = ("*", self.name)
+        if isinstance(getattr(component, "model", None), Model):
+            for node in component.model.walk():
+                if isinstance(node, Tok2VecListener) and node.upstream_name in names:
+                    self.add_listener(node, component.name)
+
+    def predict(self, docs: Iterable[Doc]):
+        """Apply the pipeline's model to a batch of docs, without modifying them.
+        Returns a single tensor for a batch of documents.
+
+        docs (Iterable[Doc]): The documents to predict.
+        RETURNS: Vector representations for each token in the documents.
+
+        DOCS: https://spacy.io/api/tok2vec#predict
+        """
+        tokvecs = self.model.predict(docs)
+        batch_id = Tok2VecListener.get_batch_id(docs)
+        for listener in self.listeners:
+            listener.receive(batch_id, tokvecs, _empty_backprop)
+        return tokvecs
+
+    def set_annotations(self, docs: Sequence[Doc], tokvecses) -> None:
+        """Modify a batch of documents, using pre-computed scores.
+
+        docs (Iterable[Doc]): The documents to modify.
+        tokvecses: The tensors to set, produced by Tok2Vec.predict.
+
+        DOCS: https://spacy.io/api/tok2vec#set_annotations
+        """
+        for doc, tokvecs in zip(docs, tokvecses):
+            assert tokvecs.shape[0] == len(doc)
+            doc.tensor = tokvecs
+
+    def update(
+        self,
+        examples: Iterable[Example],
+        *,
+        drop: float = 0.0,
+        sgd: Optional[Optimizer] = None,
+        losses: Optional[Dict[str, float]] = None,
+    ):
+        """Learn from a batch of documents and gold-standard information,
+        updating the pipe's model.
+
+        examples (Iterable[Example]): A batch of Example objects.
+        drop (float): The dropout rate.
+        sgd (thinc.api.Optimizer): The optimizer.
+        losses (Dict[str, float]): Optional record of the loss during training.
+            Updated using the component name as the key.
+        RETURNS (Dict[str, float]): The updated losses dictionary.
+
+        DOCS: https://spacy.io/api/tok2vec#update
+        """
+        if losses is None:
+            losses = {}
+        validate_examples(examples, "Tok2Vec.update")
+        docs = [eg.predicted for eg in examples]
+        set_dropout_rate(self.model, drop)
+        tokvecs, bp_tokvecs = self.model.begin_update(docs)
+        d_tokvecs = [self.model.ops.alloc2f(*t2v.shape) for t2v in tokvecs]
+        losses.setdefault(self.name, 0.0)
+
+        def accumulate_gradient(one_d_tokvecs):
+            """Accumulate tok2vec loss and gradient. This is passed as a callback
+            to all but the last listener. Only the last one does the backprop.
+            """
+            nonlocal d_tokvecs
+            for i in range(len(one_d_tokvecs)):
+                d_tokvecs[i] += one_d_tokvecs[i]
+                losses[self.name] += float((one_d_tokvecs[i] ** 2).sum())
+            return [self.model.ops.alloc2f(*t2v.shape) for t2v in tokvecs]
+
+        def backprop(one_d_tokvecs):
+            """Callback to actually do the backprop. Passed to last listener."""
+            accumulate_gradient(one_d_tokvecs)
+            d_docs = bp_tokvecs(d_tokvecs)
+            if sgd is not None:
+                self.finish_update(sgd)
+            return d_docs
+
+        batch_id = Tok2VecListener.get_batch_id(docs)
+        for listener in self.listeners[:-1]:
+            listener.receive(batch_id, tokvecs, accumulate_gradient)
+        if self.listeners:
+            self.listeners[-1].receive(batch_id, tokvecs, backprop)
+        return losses
+
+    def get_loss(self, examples, scores) -> None:
+        pass
+
+    def initialize(
+        self,
+        get_examples: Callable[[], Iterable[Example]],
+        *,
+        nlp: Optional[Language] = None,
+    ):
+        """Initialize the pipe for training, using a representative set
+        of data examples.
+
+        get_examples (Callable[[], Iterable[Example]]): Function that
+            returns a representative sample of gold-standard Example objects.
+        nlp (Language): The current nlp object the component is part of.
+
+        DOCS: https://spacy.io/api/tok2vec#initialize
+        """
+        validate_get_examples(get_examples, "Tok2Vec.initialize")
+        doc_sample = []
+        for example in islice(get_examples(), 10):
+            doc_sample.append(example.x)
+        assert doc_sample, Errors.E923.format(name=self.name)
+        self.model.initialize(X=doc_sample)
+
+    def add_label(self, label):
+        raise NotImplementedError
+
+
+class Tok2VecListener(Model):
+    """A layer that gets fed its answers from an upstream connection,
+    for instance from a component earlier in the pipeline.
+
+    The Tok2VecListener layer is used as a sublayer within a component such
+    as a parser, NER or text categorizer. Usually you'll have multiple listeners
+    connecting to a single upstream Tok2Vec component, that's earlier in the
+    pipeline. The Tok2VecListener layers act as proxies, passing the predictions
+    from the Tok2Vec component into downstream components, and communicating
+    gradients back upstream.
+    """
+
+    name = "tok2vec-listener"
+
+    def __init__(self, upstream_name: str, width: int) -> None:
+        """
+        upstream_name (str): A string to identify the 'upstream' Tok2Vec component
+            to communicate with. The upstream name should either be the wildcard
+            string '*', or the name of the `Tok2Vec` component. You'll almost
+            never have multiple upstream Tok2Vec components, so the wildcard
+            string will almost always be fine.
+        width (int):
+            The width of the vectors produced by the upstream tok2vec component.
+        """
+        Model.__init__(self, name=self.name, forward=forward, dims={"nO": width})
+        self.upstream_name = upstream_name
+        self._batch_id: Optional[int] = None
+        self._outputs = None
+        self._backprop = None
+
+    @classmethod
+    def get_batch_id(cls, inputs: Iterable[Doc]) -> int:
+        """Calculate a content-sensitive hash of the batch of documents, to check
+        whether the next batch of documents is unexpected.
+        """
+        return sum(sum(token.orth for token in doc) for doc in inputs)
+
+    def receive(self, batch_id: int, outputs, backprop) -> None:
+        """Store a batch of training predictions and a backprop callback. The
+        predictions and callback are produced by the upstream Tok2Vec component,
+        and later will be used when the listener's component's model is called.
+        """
+        self._batch_id = batch_id
+        self._outputs = outputs
+        self._backprop = backprop
+
+    def verify_inputs(self, inputs) -> bool:
+        """Check that the batch of Doc objects matches the ones we have a
+        prediction for.
+        """
+        if self._batch_id is None and self._outputs is None:
+            raise ValueError(Errors.E954)
+        else:
+            batch_id = self.get_batch_id(inputs)
+            if batch_id != self._batch_id:
+                raise ValueError(Errors.E953.format(id1=batch_id, id2=self._batch_id))
+            else:
+                return True
+
+
+def forward(model: Tok2VecListener, inputs, is_train: bool):
+    """Supply the outputs from the upstream Tok2Vec component."""
+    if is_train:
+        model.verify_inputs(inputs)
+        return model._outputs, model._backprop
+    else:
+        # This is pretty grim, but it's hard to do better :(.
+        # It's hard to avoid relying on the doc.tensor attribute, because the
+        # pipeline components can batch the data differently during prediction.
+        # That doesn't happen in update, where the nlp object works on batches
+        # of data.
+        # When the components batch differently, we don't receive a matching
+        # prediction from the upstream, so we can't predict.
+        outputs = []
+        width = model.get_dim("nO")
+        for doc in inputs:
+            if doc.tensor.size == 0:
+                # But we do need to do *something* if the tensor hasn't been set.
+                # The compromise is to at least return data of the right shape,
+                # so the output is valid.
+                outputs.append(model.ops.alloc2f(len(doc), width))
+            else:
+                outputs.append(doc.tensor)
+        return outputs, lambda dX: []
+
+
+def _empty_backprop(dX):  # for pickling
+    return []

cgpenv/Lib/site-packages/spacy/pipeline/trainable_pipe.pxd

+from .pipe cimport Pipe
+from ..vocab cimport Vocab
+
+cdef class TrainablePipe(Pipe):
+    cdef public Vocab vocab
+    cdef public object model
+    cdef public object cfg
+    cdef public object scorer

cgpenv/Lib/site-packages/spacy/pipeline/trainable_pipe.pyx

+# cython: infer_types=True, profile=True
+from typing import Iterable, Iterator, Optional, Dict, Tuple, Callable
+import srsly
+from thinc.api import set_dropout_rate, Model, Optimizer
+
+from ..tokens.doc cimport Doc
+
+from ..training import validate_examples
+from ..errors import Errors
+from .pipe import Pipe, deserialize_config
+from .. import util
+from ..vocab import Vocab
+from ..language import Language
+from ..training import Example
+
+
+cdef class TrainablePipe(Pipe):
+    """This class is a base class and not instantiated directly. Trainable
+    pipeline components like the EntityRecognizer or TextCategorizer inherit
+    from it and it defines the interface that components should follow to
+    function as trainable components in a spaCy pipeline.
+
+    DOCS: https://spacy.io/api/pipe
+    """
+    def __init__(self, vocab: Vocab, model: Model, name: str, **cfg):
+        """Initialize a pipeline component.
+
+        vocab (Vocab): The shared vocabulary.
+        model (thinc.api.Model): The Thinc Model powering the pipeline component.
+        name (str): The component instance name.
+        **cfg: Additional settings and config parameters.
+
+        DOCS: https://spacy.io/api/pipe#init
+        """
+        self.vocab = vocab
+        self.model = model
+        self.name = name
+        self.cfg = dict(cfg)
+
+    def __call__(self, Doc doc) -> Doc:
+        """Apply the pipe to one document. The document is modified in place,
+        and returned. This usually happens under the hood when the nlp object
+        is called on a text and all components are applied to the Doc.
+
+        docs (Doc): The Doc to process.
+        RETURNS (Doc): The processed Doc.
+
+        DOCS: https://spacy.io/api/pipe#call
+        """
+        error_handler = self.get_error_handler()
+        try:
+            scores = self.predict([doc])
+            self.set_annotations([doc], scores)
+            return doc
+        except Exception as e:
+            error_handler(self.name, self, [doc], e)
+
+    def pipe(self, stream: Iterable[Doc], *, batch_size: int=128) -> Iterator[Doc]:
+        """Apply the pipe to a stream of documents. This usually happens under
+        the hood when the nlp object is called on a text and all components are
+        applied to the Doc.
+
+        stream (Iterable[Doc]): A stream of documents.
+        batch_size (int): The number of documents to buffer.
+        error_handler (Callable[[str, List[Doc], Exception], Any]): Function that
+            deals with a failing batch of documents. The default function just reraises
+            the exception.
+        YIELDS (Doc): Processed documents in order.
+
+        DOCS: https://spacy.io/api/pipe#pipe
+        """
+        error_handler = self.get_error_handler()
+        for docs in util.minibatch(stream, size=batch_size):
+            try:
+                scores = self.predict(docs)
+                self.set_annotations(docs, scores)
+                yield from docs
+            except Exception as e:
+                error_handler(self.name, self, docs, e)
+
+    def predict(self, docs: Iterable[Doc]):
+        """Apply the pipeline's model to a batch of docs, without modifying them.
+        Returns a single tensor for a batch of documents.
+
+        docs (Iterable[Doc]): The documents to predict.
+        RETURNS: Vector representations of the predictions.
+
+        DOCS: https://spacy.io/api/pipe#predict
+        """
+        raise NotImplementedError(Errors.E931.format(parent="TrainablePipe", method="predict", name=self.name))
+
+    def set_annotations(self, docs: Iterable[Doc], scores):
+        """Modify a batch of documents, using pre-computed scores.
+
+        docs (Iterable[Doc]): The documents to modify.
+        scores: The scores to assign.
+
+        DOCS: https://spacy.io/api/pipe#set_annotations
+        """
+        raise NotImplementedError(Errors.E931.format(parent="TrainablePipe", method="set_annotations", name=self.name))
+
+    def update(self,
+               examples: Iterable["Example"],
+               *,
+               drop: float=0.0,
+               sgd: Optimizer=None,
+               losses: Optional[Dict[str, float]]=None) -> Dict[str, float]:
+        """Learn from a batch of documents and gold-standard information,
+        updating the pipe's model. Delegates to predict and get_loss.
+
+        examples (Iterable[Example]): A batch of Example objects.
+        drop (float): The dropout rate.
+        sgd (thinc.api.Optimizer): The optimizer.
+        losses (Dict[str, float]): Optional record of the loss during training.
+            Updated using the component name as the key.
+        RETURNS (Dict[str, float]): The updated losses dictionary.
+
+        DOCS: https://spacy.io/api/pipe#update
+        """
+        if losses is None:
+            losses = {}
+        if not hasattr(self, "model") or self.model in (None, True, False):
+            return losses
+        losses.setdefault(self.name, 0.0)
+        validate_examples(examples, "TrainablePipe.update")
+        if not any(len(eg.predicted) if eg.predicted else 0 for eg in examples):
+            # Handle cases where there are no tokens in any docs.
+            return losses
+        set_dropout_rate(self.model, drop)
+        scores, bp_scores = self.model.begin_update([eg.predicted for eg in examples])
+        loss, d_scores = self.get_loss(examples, scores)
+        bp_scores(d_scores)
+        if sgd not in (None, False):
+            self.finish_update(sgd)
+        losses[self.name] += loss
+        return losses
+
+    def rehearse(self,
+                 examples: Iterable[Example],
+                 *,
+                 sgd: Optimizer=None,
+                 losses: Dict[str, float]=None,
+                 **config) -> Dict[str, float]:
+        """Perform a "rehearsal" update from a batch of data. Rehearsal updates
+        teach the current model to make predictions similar to an initial model,
+        to try to address the "catastrophic forgetting" problem. This feature is
+        experimental.
+
+        examples (Iterable[Example]): A batch of Example objects.
+        sgd (thinc.api.Optimizer): The optimizer.
+        losses (Dict[str, float]): Optional record of the loss during training.
+            Updated using the component name as the key.
+        RETURNS (Dict[str, float]): The updated losses dictionary.
+
+        DOCS: https://spacy.io/api/pipe#rehearse
+        """
+        pass
+
+    def get_loss(self, examples: Iterable[Example], scores) -> Tuple[float, float]:
+        """Find the loss and gradient of loss for the batch of documents and
+        their predicted scores.
+
+        examples (Iterable[Examples]): The batch of examples.
+        scores: Scores representing the model's predictions.
+        RETURNS (Tuple[float, float]): The loss and the gradient.
+
+        DOCS: https://spacy.io/api/pipe#get_loss
+        """
+        raise NotImplementedError(Errors.E931.format(parent="TrainablePipe", method="get_loss", name=self.name))
+
+    def create_optimizer(self) -> Optimizer:
+        """Create an optimizer for the pipeline component.
+
+        RETURNS (thinc.api.Optimizer): The optimizer.
+
+        DOCS: https://spacy.io/api/pipe#create_optimizer
+        """
+        return util.create_default_optimizer()
+
+    def initialize(self, get_examples: Callable[[], Iterable[Example]], *, nlp: Language=None):
+        """Initialize the pipe for training, using data examples if available.
+        This method needs to be implemented by each TrainablePipe component,
+        ensuring the internal model (if available) is initialized properly
+        using the provided sample of Example objects.
+
+        get_examples (Callable[[], Iterable[Example]]): Function that
+            returns a representative sample of gold-standard Example objects.
+        nlp (Language): The current nlp object the component is part of.
+
+        DOCS: https://spacy.io/api/pipe#initialize
+        """
+        raise NotImplementedError(Errors.E931.format(parent="TrainablePipe", method="initialize", name=self.name))
+
+    def add_label(self, label: str) -> int:
+        """Add an output label.
+        For TrainablePipe components, it is possible to
+        extend pretrained models with new labels, but care should be taken to
+        avoid the "catastrophic forgetting" problem.
+
+        label (str): The label to add.
+        RETURNS (int): 0 if label is already present, otherwise 1.
+
+        DOCS: https://spacy.io/api/pipe#add_label
+        """
+        raise NotImplementedError(Errors.E931.format(parent="Pipe", method="add_label", name=self.name))
+
+    @property
+    def is_trainable(self) -> bool:
+        return True
+
+    @property
+    def is_resizable(self) -> bool:
+        return getattr(self, "model", None) and "resize_output" in self.model.attrs
+
+    def _allow_extra_label(self) -> None:
+        """Raise an error if the component can not add any more labels."""
+        nO = None
+        if self.model.has_dim("nO"):
+            nO = self.model.get_dim("nO")
+        elif self.model.has_ref("output_layer") and self.model.get_ref("output_layer").has_dim("nO"):
+            nO = self.model.get_ref("output_layer").get_dim("nO")
+        if nO is not None and nO == len(self.labels):
+            if not self.is_resizable:
+                raise ValueError(Errors.E922.format(name=self.name, nO=self.model.get_dim("nO")))
+
+    def set_output(self, nO: int) -> None:
+        if self.is_resizable:
+            self.model.attrs["resize_output"](self.model, nO)
+        else:
+            raise NotImplementedError(Errors.E921)
+
+    def use_params(self, params: dict):
+        """Modify the pipe's model, to use the given parameter values. At the
+        end of the context, the original parameters are restored.
+
+        params (dict): The parameter values to use in the model.
+
+        DOCS: https://spacy.io/api/pipe#use_params
+        """
+        with self.model.use_params(params):
+            yield
+
+    def finish_update(self, sgd: Optimizer) -> None:
+        """Update parameters using the current parameter gradients.
+        The Optimizer instance contains the functionality to perform
+        the stochastic gradient descent.
+
+        sgd (thinc.api.Optimizer): The optimizer.
+
+        DOCS: https://spacy.io/api/pipe#finish_update
+        """
+        self.model.finish_update(sgd)
+
+    def _validate_serialization_attrs(self):
+        """Check that the pipe implements the required attributes. If a subclass
+        implements a custom __init__ method but doesn't set these attributes,
+        they currently default to None, so we need to perform additonal checks.
+        """
+        if not hasattr(self, "vocab") or self.vocab is None:
+            raise ValueError(Errors.E899.format(name=util.get_object_name(self)))
+        if not hasattr(self, "model") or self.model is None:
+            raise ValueError(Errors.E898.format(name=util.get_object_name(self)))
+
+    def to_bytes(self, *, exclude=tuple()):
+        """Serialize the pipe to a bytestring.
+
+        exclude (Iterable[str]): String names of serialization fields to exclude.
+        RETURNS (bytes): The serialized object.
+
+        DOCS: https://spacy.io/api/pipe#to_bytes
+        """
+        self._validate_serialization_attrs()
+        serialize = {}
+        if hasattr(self, "cfg") and self.cfg is not None:
+            serialize["cfg"] = lambda: srsly.json_dumps(self.cfg)
+        serialize["vocab"] = lambda: self.vocab.to_bytes(exclude=exclude)
+        serialize["model"] = self.model.to_bytes
+        return util.to_bytes(serialize, exclude)
+
+    def from_bytes(self, bytes_data, *, exclude=tuple()):
+        """Load the pipe from a bytestring.
+
+        exclude (Iterable[str]): String names of serialization fields to exclude.
+        RETURNS (TrainablePipe): The loaded object.
+
+        DOCS: https://spacy.io/api/pipe#from_bytes
+        """
+        self._validate_serialization_attrs()
+
+        def load_model(b):
+            try:
+                self.model.from_bytes(b)
+            except AttributeError:
+                raise ValueError(Errors.E149) from None
+
+        deserialize = {}
+        if hasattr(self, "cfg") and self.cfg is not None:
+            deserialize["cfg"] = lambda b: self.cfg.update(srsly.json_loads(b))
+        deserialize["vocab"] = lambda b: self.vocab.from_bytes(b, exclude=exclude)
+        deserialize["model"] = load_model
+        util.from_bytes(bytes_data, deserialize, exclude)
+        return self
+
+    def to_disk(self, path, *, exclude=tuple()):
+        """Serialize the pipe to disk.
+
+        path (str / Path): Path to a directory.
+        exclude (Iterable[str]): String names of serialization fields to exclude.
+
+        DOCS: https://spacy.io/api/pipe#to_disk
+        """
+        self._validate_serialization_attrs()
+        serialize = {}
+        if hasattr(self, "cfg") and self.cfg is not None:
+            serialize["cfg"] = lambda p: srsly.write_json(p, self.cfg)
+        serialize["vocab"] = lambda p: self.vocab.to_disk(p, exclude=exclude)
+        serialize["model"] = lambda p: self.model.to_disk(p)
+        util.to_disk(path, serialize, exclude)
+
+    def from_disk(self, path, *, exclude=tuple()):
+        """Load the pipe from disk.
+
+        path (str / Path): Path to a directory.
+        exclude (Iterable[str]): String names of serialization fields to exclude.
+        RETURNS (TrainablePipe): The loaded object.
+
+        DOCS: https://spacy.io/api/pipe#from_disk
+        """
+        self._validate_serialization_attrs()
+
+        def load_model(p):
+            try:
+                with open(p, "rb") as mfile:
+                    self.model.from_bytes(mfile.read())
+            except AttributeError:
+                raise ValueError(Errors.E149) from None
+
+        deserialize = {}
+        if hasattr(self, "cfg") and self.cfg is not None:
+            deserialize["cfg"] = lambda p: self.cfg.update(deserialize_config(p))
+        deserialize["vocab"] = lambda p: self.vocab.from_disk(p, exclude=exclude)
+        deserialize["model"] = load_model
+        util.from_disk(path, deserialize, exclude)
+        return self

cgpenv/Lib/site-packages/spacy/pipeline/transition_parser.pxd

+from cymem.cymem cimport Pool
+
+from ..vocab cimport Vocab
+from .trainable_pipe cimport TrainablePipe
+from ._parser_internals.transition_system cimport Transition, TransitionSystem
+from ._parser_internals._state cimport StateC
+from ..ml.parser_model cimport WeightsC, ActivationsC, SizesC
+
+
+cdef class Parser(TrainablePipe):
+    cdef public object _rehearsal_model
+    cdef readonly TransitionSystem moves
+    cdef public object _multitasks
+
+    cdef void _parseC(self, StateC** states,
+            WeightsC weights, SizesC sizes) nogil
+
+    cdef void c_transition_batch(self, StateC** states, const float* scores,
+            int nr_class, int batch_size) nogil

cgpenv/Lib/site-packages/spacy/pipeline/transition_parser.pyx

+# cython: infer_types=True, cdivision=True, boundscheck=False, binding=True
+from __future__ import print_function
+from cymem.cymem cimport Pool
+cimport numpy as np
+from itertools import islice
+from libcpp.vector cimport vector
+from libc.string cimport memset, memcpy
+from libc.stdlib cimport calloc, free
+import random
+
+import srsly
+from thinc.api import set_dropout_rate, CupyOps
+from thinc.extra.search cimport Beam
+import numpy.random
+import numpy
+import warnings
+
+from ._parser_internals.stateclass cimport StateClass
+from ..ml.parser_model cimport alloc_activations, free_activations
+from ..ml.parser_model cimport predict_states, arg_max_if_valid
+from ..ml.parser_model cimport WeightsC, ActivationsC, SizesC, cpu_log_loss
+from ..ml.parser_model cimport get_c_weights, get_c_sizes
+from ..tokens.doc cimport Doc
+from .trainable_pipe import TrainablePipe
+from ._parser_internals cimport _beam_utils
+from ._parser_internals import _beam_utils
+
+from ..training import validate_examples, validate_get_examples
+from ..errors import Errors, Warnings
+from .. import util
+
+
+cdef class Parser(TrainablePipe):
+    """
+    Base class of the DependencyParser and EntityRecognizer.
+    """
+
+    def __init__(
+        self,
+        Vocab vocab,
+        model,
+        name="base_parser",
+        moves=None,
+        *,
+        update_with_oracle_cut_size,
+        min_action_freq,
+        learn_tokens,
+        beam_width=1,
+        beam_density=0.0,
+        beam_update_prob=0.0,
+        multitasks=tuple(),
+        incorrect_spans_key=None,
+        scorer=None,
+    ):
+        """Create a Parser.
+
+        vocab (Vocab): The vocabulary object. Must be shared with documents
+            to be processed. The value is set to the `.vocab` attribute.
+        model (Model): The model for the transition-based parser. The model needs
+            to have a specific substructure of named components --- see the
+            spacy.ml.tb_framework.TransitionModel for details.
+        name (str): The name of the pipeline component
+        moves (Optional[TransitionSystem]): This defines how the parse-state is created,
+            updated and evaluated. If 'moves' is None, a new instance is
+            created with `self.TransitionSystem()`. Defaults to `None`.
+        update_with_oracle_cut_size (int): During training, cut long sequences into
+            shorter segments by creating intermediate states based on the gold-standard
+            history. The model is not very sensitive to this parameter, so you usually
+            won't need to change it. 100 is a good default.
+        min_action_freq (int): The minimum frequency of labelled actions to retain.
+            Rarer labelled actions have their label backed-off to "dep". While this
+            primarily affects the label accuracy, it can also affect the attachment
+            structure, as the labels are used to represent the pseudo-projectivity
+            transformation.
+        learn_tokens (bool): Whether to learn to merge subtokens that are split
+            relative to the gold standard. Experimental.
+        beam_width (int): The number of candidate analyses to maintain.
+        beam_density (float): The minimum ratio between the scores of the first and
+            last candidates in the beam. This allows the parser to avoid exploring
+            candidates that are too far behind. This is mostly intended to improve
+            efficiency, but it can also improve accuracy as deeper search is not
+            always better.
+        beam_update_prob (float): The chance of making a beam update, instead of a
+            greedy update. Greedy updates are an approximation for the beam updates,
+            and are faster to compute.
+        multitasks: additional multi-tasking components. Experimental.
+        incorrect_spans_key (Optional[str]): Identifies spans that are known
+            to be incorrect entity annotations. The incorrect entity annotations
+            can be stored in the span group, under this key.
+        scorer (Optional[Callable]): The scoring method. Defaults to None.
+        """
+        self.vocab = vocab
+        self.name = name
+        cfg = {
+            "moves": moves,
+            "update_with_oracle_cut_size": update_with_oracle_cut_size,
+            "multitasks": list(multitasks),
+            "min_action_freq": min_action_freq,
+            "learn_tokens": learn_tokens,
+            "beam_width": beam_width,
+            "beam_density": beam_density,
+            "beam_update_prob": beam_update_prob,
+            "incorrect_spans_key": incorrect_spans_key
+        }
+        if moves is None:
+            # EntityRecognizer -> BiluoPushDown
+            # DependencyParser -> ArcEager
+            moves = self.TransitionSystem(
+                self.vocab.strings,
+                incorrect_spans_key=incorrect_spans_key
+            )
+        self.moves = moves
+        self.model = model
+        if self.moves.n_moves != 0:
+            self.set_output(self.moves.n_moves)
+        self.cfg = cfg
+        self._multitasks = []
+        for multitask in cfg["multitasks"]:
+            self.add_multitask_objective(multitask)
+
+        self._rehearsal_model = None
+        self.scorer = scorer
+
+    def __getnewargs_ex__(self):
+        """This allows pickling the Parser and its keyword-only init arguments"""
+        args = (self.vocab, self.model, self.name, self.moves)
+        return args, self.cfg
+
+    @property
+    def move_names(self):
+        names = []
+        for i in range(self.moves.n_moves):
+            name = self.moves.move_name(self.moves.c[i].move, self.moves.c[i].label)
+            # Explicitly removing the internal "U-" token used for blocking entities
+            if name != "U-":
+                names.append(name)
+        return names
+
+    @property
+    def labels(self):
+        class_names = [self.moves.get_class_name(i) for i in range(self.moves.n_moves)]
+        return class_names
+
+    @property
+    def label_data(self):
+        return self.moves.labels
+
+    @property
+    def tok2vec(self):
+        """Return the embedding and convolutional layer of the model."""
+        return self.model.get_ref("tok2vec")
+
+    @property
+    def postprocesses(self):
+        # Available for subclasses, e.g. to deprojectivize
+        return []
+
+    @property
+    def incorrect_spans_key(self):
+        return self.cfg["incorrect_spans_key"]
+
+    def add_label(self, label):
+        resized = False
+        for action in self.moves.action_types:
+            added = self.moves.add_action(action, label)
+            if added:
+                resized = True
+        if resized:
+            self._resize()
+            self.vocab.strings.add(label)
+            return 1
+        return 0
+
+    def _ensure_labels_are_added(self, docs):
+        """Ensure that all labels for a batch of docs are added."""
+        resized = False
+        labels = set()
+        for doc in docs:
+            labels.update(self.moves.get_doc_labels(doc))
+        for label in labels:
+            for action in self.moves.action_types:
+                added = self.moves.add_action(action, label)
+                if added:
+                    self.vocab.strings.add(label)
+                    resized = True
+        if resized:
+            self._resize()
+            return 1
+        return 0
+
+    def _resize(self):
+        self.model.attrs["resize_output"](self.model, self.moves.n_moves)
+        if self._rehearsal_model not in (True, False, None):
+            self._rehearsal_model.attrs["resize_output"](
+                self._rehearsal_model, self.moves.n_moves
+            )
+
+    def add_multitask_objective(self, target):
+        # Defined in subclasses, to avoid circular import
+        raise NotImplementedError
+
+    def init_multitask_objectives(self, get_examples, pipeline, **cfg):
+        """Setup models for secondary objectives, to benefit from multi-task
+        learning. This method is intended to be overridden by subclasses.
+
+        For instance, the dependency parser can benefit from sharing
+        an input representation with a label prediction model. These auxiliary
+        models are discarded after training.
+        """
+        pass
+
+    def use_params(self, params):
+        # Can't decorate cdef class :(. Workaround.
+        with self.model.use_params(params):
+            yield
+
+    def pipe(self, docs, *, int batch_size=256):
+        """Process a stream of documents.
+
+        stream: The sequence of documents to process.
+        batch_size (int): Number of documents to accumulate into a working set.
+        error_handler (Callable[[str, List[Doc], Exception], Any]): Function that
+            deals with a failing batch of documents. The default function just reraises
+            the exception.
+
+        YIELDS (Doc): Documents, in order.
+        """
+        cdef Doc doc
+        error_handler = self.get_error_handler()
+        for batch in util.minibatch(docs, size=batch_size):
+            batch_in_order = list(batch)
+            try:
+                by_length = sorted(batch, key=lambda doc: len(doc))
+                for subbatch in util.minibatch(by_length, size=max(batch_size//4, 2)):
+                    subbatch = list(subbatch)
+                    parse_states = self.predict(subbatch)
+                    self.set_annotations(subbatch, parse_states)
+                yield from batch_in_order
+            except Exception as e:
+                error_handler(self.name, self, batch_in_order, e)
+
+
+    def predict(self, docs):
+        if isinstance(docs, Doc):
+            docs = [docs]
+        if not any(len(doc) for doc in docs):
+            result = self.moves.init_batch(docs)
+            return result
+        if self.cfg["beam_width"] == 1:
+            return self.greedy_parse(docs, drop=0.0)
+        else:
+            return self.beam_parse(
+                docs,
+                drop=0.0,
+                beam_width=self.cfg["beam_width"],
+                beam_density=self.cfg["beam_density"]
+            )
+
+    def greedy_parse(self, docs, drop=0.):
+        cdef vector[StateC*] states
+        cdef StateClass state
+        self._ensure_labels_are_added(docs)
+        set_dropout_rate(self.model, drop)
+        batch = self.moves.init_batch(docs)
+        model = self.model.predict(docs)
+        weights = get_c_weights(model)
+        for state in batch:
+            if not state.is_final():
+                states.push_back(state.c)
+        sizes = get_c_sizes(model, states.size())
+        with nogil:
+            self._parseC(&states[0],
+                weights, sizes)
+        model.clear_memory()
+        del model
+        return batch
+
+    def beam_parse(self, docs, int beam_width, float drop=0., beam_density=0.):
+        cdef Beam beam
+        cdef Doc doc
+        self._ensure_labels_are_added(docs)
+        batch = _beam_utils.BeamBatch(
+            self.moves,
+            self.moves.init_batch(docs),
+            None,
+            beam_width,
+            density=beam_density
+        )
+        model = self.model.predict(docs)
+        while not batch.is_done:
+            states = batch.get_unfinished_states()
+            if not states:
+                break
+            scores = model.predict(states)
+            batch.advance(scores)
+        model.clear_memory()
+        del model
+        return list(batch)
+
+    cdef void _parseC(self, StateC** states,
+            WeightsC weights, SizesC sizes) nogil:
+        cdef int i, j
+        cdef vector[StateC*] unfinished
+        cdef ActivationsC activations = alloc_activations(sizes)
+        while sizes.states >= 1:
+            predict_states(&activations,
+                states, &weights, sizes)
+            # Validate actions, argmax, take action.
+            self.c_transition_batch(states,
+                activations.scores, sizes.classes, sizes.states)
+            for i in range(sizes.states):
+                if not states[i].is_final():
+                    unfinished.push_back(states[i])
+            for i in range(unfinished.size()):
+                states[i] = unfinished[i]
+            sizes.states = unfinished.size()
+            unfinished.clear()
+        free_activations(&activations)
+
+    def set_annotations(self, docs, states_or_beams):
+        cdef StateClass state
+        cdef Beam beam
+        cdef Doc doc
+        states = _beam_utils.collect_states(states_or_beams, docs)
+        for i, (state, doc) in enumerate(zip(states, docs)):
+            self.moves.set_annotations(state, doc)
+            for hook in self.postprocesses:
+                hook(doc)
+
+    def transition_states(self, states, float[:, ::1] scores):
+        cdef StateClass state
+        cdef float* c_scores = &scores[0, 0]
+        cdef vector[StateC*] c_states
+        for state in states:
+            c_states.push_back(state.c)
+        self.c_transition_batch(&c_states[0], c_scores, scores.shape[1], scores.shape[0])
+        return [state for state in states if not state.c.is_final()]
+
+    cdef void c_transition_batch(self, StateC** states, const float* scores,
+            int nr_class, int batch_size) nogil:
+        # n_moves should not be zero at this point, but make sure to avoid zero-length mem alloc
+        with gil:
+            assert self.moves.n_moves > 0, Errors.E924.format(name=self.name)
+        is_valid = <int*>calloc(self.moves.n_moves, sizeof(int))
+        cdef int i, guess
+        cdef Transition action
+        for i in range(batch_size):
+            self.moves.set_valid(is_valid, states[i])
+            guess = arg_max_if_valid(&scores[i*nr_class], is_valid, nr_class)
+            if guess == -1:
+                # This shouldn't happen, but it's hard to raise an error here,
+                # and we don't want to infinite loop. So, force to end state.
+                states[i].force_final()
+            else:
+                action = self.moves.c[guess]
+                action.do(states[i], action.label)
+        free(is_valid)
+
+    def update(self, examples, *, drop=0., sgd=None, losses=None):
+        cdef StateClass state
+        if losses is None:
+            losses = {}
+        losses.setdefault(self.name, 0.)
+        validate_examples(examples, "Parser.update")
+        self._ensure_labels_are_added(
+            [eg.x for eg in examples] + [eg.y for eg in examples]
+        )
+        for multitask in self._multitasks:
+            multitask.update(examples, drop=drop, sgd=sgd)
+        n_examples = len([eg for eg in examples if self.moves.has_gold(eg)])
+        if n_examples == 0:
+            return losses
+        set_dropout_rate(self.model, drop)
+        # The probability we use beam update, instead of falling back to
+        # a greedy update
+        beam_update_prob = self.cfg["beam_update_prob"]
+        if self.cfg['beam_width'] >= 2 and numpy.random.random() < beam_update_prob:
+            return self.update_beam(
+                examples,
+                beam_width=self.cfg["beam_width"],
+                sgd=sgd,
+                losses=losses,
+                beam_density=self.cfg["beam_density"]
+            )
+        max_moves = self.cfg["update_with_oracle_cut_size"]
+        if max_moves >= 1:
+            # Chop sequences into lengths of this many words, to make the
+            # batch uniform length.
+            max_moves = int(random.uniform(max_moves // 2, max_moves * 2))
+            states, golds, _ = self._init_gold_batch(
+                examples,
+                max_length=max_moves
+            )
+        else:
+            states, golds, _ = self.moves.init_gold_batch(examples)
+        if not states:
+            return losses
+        model, backprop_tok2vec = self.model.begin_update([eg.x for eg in examples])
+ 
+        all_states = list(states)
+        states_golds = list(zip(states, golds))
+        n_moves = 0
+        while states_golds:
+            states, golds = zip(*states_golds)
+            scores, backprop = model.begin_update(states)
+            d_scores = self.get_batch_loss(states, golds, scores, losses)
+            # Note that the gradient isn't normalized by the batch size
+            # here, because our "samples" are really the states...But we
+            # can't normalize by the number of states either, as then we'd
+            # be getting smaller gradients for states in long sequences.
+            backprop(d_scores)
+            # Follow the predicted action
+            self.transition_states(states, scores)
+            states_golds = [(s, g) for (s, g) in zip(states, golds) if not s.is_final()]
+            if max_moves >= 1 and n_moves >= max_moves:
+                break
+            n_moves += 1
+
+        backprop_tok2vec(golds)
+        if sgd not in (None, False):
+            self.finish_update(sgd)
+        # Ugh, this is annoying. If we're working on GPU, we want to free the
+        # memory ASAP. It seems that Python doesn't necessarily get around to
+        # removing these in time if we don't explicitly delete? It's confusing.
+        del backprop
+        del backprop_tok2vec
+        model.clear_memory()
+        del model
+        return losses
+
+    def rehearse(self, examples, sgd=None, losses=None, **cfg):
+        """Perform a "rehearsal" update, to prevent catastrophic forgetting."""
+        if losses is None:
+            losses = {}
+        for multitask in self._multitasks:
+            if hasattr(multitask, 'rehearse'):
+                multitask.rehearse(examples, losses=losses, sgd=sgd)
+        if self._rehearsal_model is None:
+            return None
+        losses.setdefault(self.name, 0.)
+        validate_examples(examples, "Parser.rehearse")
+        docs = [eg.predicted for eg in examples]
+        states = self.moves.init_batch(docs)
+        # This is pretty dirty, but the NER can resize itself in init_batch,
+        # if labels are missing. We therefore have to check whether we need to
+        # expand our model output.
+        self._resize()
+        # Prepare the stepwise model, and get the callback for finishing the batch
+        set_dropout_rate(self._rehearsal_model, 0.0)
+        set_dropout_rate(self.model, 0.0)
+        tutor, _ = self._rehearsal_model.begin_update(docs)
+        model, backprop_tok2vec = self.model.begin_update(docs)
+        n_scores = 0.
+        loss = 0.
+        while states:
+            targets, _ = tutor.begin_update(states)
+            guesses, backprop = model.begin_update(states)
+            d_scores = (guesses - targets) / targets.shape[0]
+            # If all weights for an output are 0 in the original model, don't
+            # supervise that output. This allows us to add classes.
+            loss += (d_scores**2).sum()
+            backprop(d_scores)
+            # Follow the predicted action
+            self.transition_states(states, guesses)
+            states = [state for state in states if not state.is_final()]
+            n_scores += d_scores.size
+        # Do the backprop
+        backprop_tok2vec(docs)
+        if sgd is not None:
+            self.finish_update(sgd)
+        losses[self.name] += loss / n_scores
+        del backprop
+        del backprop_tok2vec
+        model.clear_memory()
+        tutor.clear_memory()
+        del model
+        del tutor
+        return losses
+
+    def update_beam(self, examples, *, beam_width,
+            drop=0., sgd=None, losses=None, beam_density=0.0):
+        states, golds, _ = self.moves.init_gold_batch(examples)
+        if not states:
+            return losses
+        # Prepare the stepwise model, and get the callback for finishing the batch
+        model, backprop_tok2vec = self.model.begin_update(
+            [eg.predicted for eg in examples])
+        loss = _beam_utils.update_beam(
+            self.moves,
+            states,
+            golds,
+            model,
+            beam_width,
+            beam_density=beam_density,
+        )
+        losses[self.name] += loss
+        backprop_tok2vec(golds)
+        if sgd is not None:
+            self.finish_update(sgd)
+
+    def get_batch_loss(self, states, golds, float[:, ::1] scores, losses):
+        cdef StateClass state
+        cdef Pool mem = Pool()
+        cdef int i
+
+        # n_moves should not be zero at this point, but make sure to avoid zero-length mem alloc
+        assert self.moves.n_moves > 0, Errors.E924.format(name=self.name)
+
+        is_valid = <int*>mem.alloc(self.moves.n_moves, sizeof(int))
+        costs = <float*>mem.alloc(self.moves.n_moves, sizeof(float))
+        cdef np.ndarray d_scores = numpy.zeros((len(states), self.moves.n_moves),
+                                        dtype='f', order='C')
+        c_d_scores = <float*>d_scores.data
+        unseen_classes = self.model.attrs["unseen_classes"]
+        for i, (state, gold) in enumerate(zip(states, golds)):
+            memset(is_valid, 0, self.moves.n_moves * sizeof(int))
+            memset(costs, 0, self.moves.n_moves * sizeof(float))
+            self.moves.set_costs(is_valid, costs, state.c, gold)
+            for j in range(self.moves.n_moves):
+                if costs[j] <= 0.0 and j in unseen_classes:
+                    unseen_classes.remove(j)
+            cpu_log_loss(c_d_scores,
+                costs, is_valid, &scores[i, 0], d_scores.shape[1])
+            c_d_scores += d_scores.shape[1]
+        # Note that we don't normalize this. See comment in update() for why.
+        if losses is not None:
+            losses.setdefault(self.name, 0.)
+            losses[self.name] += (d_scores**2).sum()
+        return d_scores
+
+    def set_output(self, nO):
+        self.model.attrs["resize_output"](self.model, nO)
+
+    def initialize(self, get_examples, nlp=None, labels=None):
+        validate_get_examples(get_examples, "Parser.initialize")
+        util.check_lexeme_norms(self.vocab, "parser or NER")
+        if labels is not None:
+            actions = dict(labels)
+        else:
+            actions = self.moves.get_actions(
+                examples=get_examples(),
+                min_freq=self.cfg['min_action_freq'],
+                learn_tokens=self.cfg["learn_tokens"]
+            )
+        for action, labels in self.moves.labels.items():
+            actions.setdefault(action, {})
+            for label, freq in labels.items():
+                if label not in actions[action]:
+                    actions[action][label] = freq
+        self.moves.initialize_actions(actions)
+        # make sure we resize so we have an appropriate upper layer
+        self._resize()
+        doc_sample = []
+        if nlp is not None:
+            for name, component in nlp.pipeline:
+                if component is self:
+                    break
+                # non-trainable components may have a pipe() implementation that refers to dummy
+                # predict and set_annotations methods
+                if hasattr(component, "pipe"):
+                    doc_sample = list(component.pipe(doc_sample, batch_size=8))
+                else:
+                    doc_sample = [component(doc) for doc in doc_sample]
+        if not doc_sample:
+            for example in islice(get_examples(), 10):
+                doc_sample.append(example.predicted)
+        assert len(doc_sample) > 0, Errors.E923.format(name=self.name)
+        self.model.initialize(doc_sample)
+        if nlp is not None:
+            self.init_multitask_objectives(get_examples, nlp.pipeline)
+
+    def to_disk(self, path, exclude=tuple()):
+        serializers = {
+            "model": lambda p: (self.model.to_disk(p) if self.model is not True else True),
+            "vocab": lambda p: self.vocab.to_disk(p, exclude=exclude),
+            "moves": lambda p: self.moves.to_disk(p, exclude=["strings"]),
+            "cfg": lambda p: srsly.write_json(p, self.cfg)
+        }
+        util.to_disk(path, serializers, exclude)
+
+    def from_disk(self, path, exclude=tuple()):
+        deserializers = {
+            "vocab": lambda p: self.vocab.from_disk(p, exclude=exclude),
+            "moves": lambda p: self.moves.from_disk(p, exclude=["strings"]),
+            "cfg": lambda p: self.cfg.update(srsly.read_json(p)),
+            "model": lambda p: None,
+        }
+        util.from_disk(path, deserializers, exclude)
+        if "model" not in exclude:
+            path = util.ensure_path(path)
+            with (path / "model").open("rb") as file_:
+                bytes_data = file_.read()
+            try:
+                self._resize()
+                self.model.from_bytes(bytes_data)
+            except AttributeError:
+                raise ValueError(Errors.E149)
+        return self
+
+    def to_bytes(self, exclude=tuple()):
+        serializers = {
+            "model": lambda: (self.model.to_bytes()),
+            "vocab": lambda: self.vocab.to_bytes(exclude=exclude),
+            "moves": lambda: self.moves.to_bytes(exclude=["strings"]),
+            "cfg": lambda: srsly.json_dumps(self.cfg, indent=2, sort_keys=True)
+        }
+        return util.to_bytes(serializers, exclude)
+
+    def from_bytes(self, bytes_data, exclude=tuple()):
+        deserializers = {
+            "vocab": lambda b: self.vocab.from_bytes(b, exclude=exclude),
+            "moves": lambda b: self.moves.from_bytes(b, exclude=["strings"]),
+            "cfg": lambda b: self.cfg.update(srsly.json_loads(b)),
+            "model": lambda b: None,
+        }
+        msg = util.from_bytes(bytes_data, deserializers, exclude)
+        if 'model' not in exclude:
+            if 'model' in msg:
+                try:
+                    self.model.from_bytes(msg['model'])
+                except AttributeError:
+                    raise ValueError(Errors.E149) from None
+        return self
+
+    def _init_gold_batch(self, examples, max_length):
+        """Make a square batch, of length equal to the shortest transition
+        sequence or a cap. A long
+        doc will get multiple states. Let's say we have a doc of length 2*N,
+        where N is the shortest doc. We'll make two states, one representing
+        long_doc[:N], and another representing long_doc[N:]."""
+        cdef:
+            StateClass start_state
+            StateClass state
+            Transition action
+        all_states = self.moves.init_batch([eg.predicted for eg in examples])
+        states = []
+        golds = []
+        to_cut = []
+        for state, eg in zip(all_states, examples):
+            if self.moves.has_gold(eg) and not state.is_final():
+                gold = self.moves.init_gold(state, eg)
+                if len(eg.x) < max_length:
+                    states.append(state)
+                    golds.append(gold)
+                else:
+                    oracle_actions = self.moves.get_oracle_sequence_from_state(
+                        state.copy(), gold)
+                    to_cut.append((eg, state, gold, oracle_actions))
+        if not to_cut:
+            return states, golds, 0
+        cdef int clas
+        for eg, state, gold, oracle_actions in to_cut:
+            for i in range(0, len(oracle_actions), max_length):
+                start_state = state.copy()
+                for clas in oracle_actions[i:i+max_length]:
+                    action = self.moves.c[clas]
+                    action.do(state.c, action.label)
+                    if state.is_final():
+                        break
+                if self.moves.has_gold(eg, start_state.B(0), state.B(0)):
+                    states.append(start_state)
+                    golds.append(gold)
+                if state.is_final():
+                    break
+        return states, golds, max_length