NLP基礎學習(二) - Text Preprocessing with spaCy

1
2
3
nlp = spacy.load("en_core_web_sm")
introduction_doc = nlp("In 1991, the World Wide Web was born. It was a medium for sharing information.")
print([token.text for token in introduction_doc])

1
2
3
file_name = "introduction.txt"
introduction_doc = nlp(pathlib.Path(file_name).read_text(encoding="utf-8"))
print([token.text for token in introduction_doc])

1
2
3
4
5
6
7
8
9
about_text = (
      "Gus Proto is a Python developer currently"
      " working for a London-based Fintech"
      " company. He is interested in learning"
      " Natural Language Processing."
)
about_doc = nlp(about_text)
sentences = list(about_doc.sents)
print(sentences)

1
2
3
print(f"{"Token":12} {"Start Index"}")
for token in about_doc:
    print(f"{token.text:12} {token.idx}")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
print(
f"{"Text with Whitespace":25}"
f"{"Is Alphanumeric?":20}"
f"{"Is Punctuation?":20}"
f"{"Is Stop Word?"}"
)

for token in about_doc:
    print(
         f"{str(token.text_with_ws):25}"
         f"{str(token.is_alpha):20}"
         f"{str(token.is_punct):20}"
         f"{str(token.is_stop)}"
    )

1
2
3
spacy_stopwords = spacy.lang.en.stop_words.STOP_WORDS
for stop_word in list(spacy_stopwords)[:10]:
    print(stop_word)

1
2
3
4
5
6
7
8
9
10
custom_about_text = (
     "Gus Proto is a Python developer currently"
     " working for a London-based Fintech"
     " company. He is interested in learning"
     " Natural Language Processing."
 )

nlp = spacy.load("en_core_web_sm")
about_doc = nlp(custom_about_text)
print([token for token in about_doc if not token.is_stop])

1
2
3
4
5
6
7
8
9
10
11
12
13
conference_help_text = (
     "Gus is helping organize a developer"
     " conference on Applications of Natural Language"
     " Processing. He keeps organizing local Python meetups"
     " and several internal talks at his workplace."
 )

nlp = spacy.load("en_core_web_sm")
conference_help_doc = nlp(conference_help_text)

for token in conference_help_doc:
    if str(token) != str(token.lemma_): # only print if lemma is different with the original token
        print(f"{str(token):>20} : {str(token.lemma_)}")

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
from collections import Counter
nlp = spacy.load("en_core_web_sm")
complete_text = (
     "Gus Proto is a Python developer currently"
     " working for a London-based Fintech company. He is"
     " interested in learning Natural Language Processing."
     " There is a developer conference happening on 21 July"
     ' 2019 in London. It is titled "Applications of Natural'
     ' Language Processing". There is a helpline number'
     " available at +44-1234567891. Gus is helping organize it."
     " He keeps organizing local Python meetups and several"
     " internal talks at his workplace. Gus is also presenting"
     ' a talk. The talk will introduce the reader about "Use'
     ' cases of Natural Language Processing in Fintech".'
     " Apart from his work, he is very passionate about music."
     " Gus is learning to play the Piano. He has enrolled"
     " himself in the weekend batch of Great Piano Academy."
     " Great Piano Academy is situated in Mayfair or the City"
     " of London and has world-class piano instructors."
 )
complete_doc = nlp(complete_text)

words = [
     token.text
     for token in complete_doc
     if not token.is_stop and not token.is_punct # filter out stop words and punctuation
]

print(Counter(words).most_common(5))

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
about_text = (
     "Gus Proto is a Python developer currently"
     " working for a London-based Fintech"
     " company. He is interested in learning"
     " Natural Language Processing."
 )

about_doc = nlp(about_text)

for token in about_doc:
    print(
        f"""
        TOKEN: {str(token)}
        =====
        POS: {token.pos_}
        EXPLANATION: {spacy.explain(token.tag_)}"""
        )

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
nlp = spacy.load("en_core_web_sm")
complete_text = (
     "Gus Proto is a Python developer currently"
     " working for a London-based Fintech company. He is"
     " interested in learning Natural Language Processing."
     " There is a developer conference happening on 21 July"
     ' 2019 in London. It is titled "Applications of Natural'
     ' Language Processing". There is a helpline number'
     " available at +44-1234567891. Gus is helping organize it."
     " He keeps organizing local Python meetups and several"
     " internal talks at his workplace. Gus is also presenting"
     ' a talk. The talk will introduce the reader about "Use'
     ' cases of Natural Language Processing in Fintech".'
     " Apart from his work, he is very passionate about music."
     " Gus is learning to play the Piano. He has enrolled"
     " himself in the weekend batch of Great Piano Academy."
     " Great Piano Academy is situated in Mayfair or the City"
     " of London and has world-class piano instructors."
 )

complete_doc = nlp(complete_text)

def is_token_allowed(token):
    return bool( token and str(token).strip() and not token.is_stop and not token.is_punct )

def preprocess_token(token):
    return token.lemma_.strip().lower()

complete_filtered_tokens = [
     preprocess_token(token)
     for token in complete_doc
     if is_token_allowed(token)
]

print(complete_filtered_tokens)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
from spacy.matcher import Matcher

nlp = spacy.load("en_core_web_sm")

about_text = (
     "Gus Proto is a Python developer currently"
     " working for a London-based Fintech"
     " company. He is interested in learning"
     " Natural Language Processing."
 )

about_doc = nlp(about_text)

matcher = Matcher(nlp.vocab)

def extract_full_name(nlp_doc):
    
    name = []
    pattern = [{"POS": "PROPN"}, {"POS": "PROPN"}] # PROPN: proper noun, 連續兩個PROPN
    matcher.add("FULL_NAME", [pattern])
    matches = matcher(nlp_doc)
    for _, start, end in matches:
        span = nlp_doc[start:end]
        name.append(span.text)
    
    return name
        
extract_full_name(about_doc)

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
nlp = spacy.load("en_core_web_sm")

piano_class_text = (
     "Great Piano Academy is situated"
     " in Mayfair or the City of London and has"
     " world-class piano instructors."
 )

piano_class_doc = nlp(piano_class_text)
for ent in piano_class_doc.ents:
    print(
        f"""
        {ent.text = }
        {ent.start_char = }
        {ent.end_char = }
        {ent.label_ = }
        spacy.explain('{ent.label_}') = {spacy.explain(ent.label_)}"""
        )

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
survey_text = (
    "Out of 5 people surveyed, James Robert,"
    " Julie Fuller and Benjamin Brooks like"
    " apples. Kelly Cox and Matthew Evans"
    " like oranges."
)

def replace_person_names(token):
    if token.ent_iob != 0 and token.ent_type_ == "PERSON":
        return "[REDACTED] "
    return token.text_with_ws

def redact_names(nlp_doc):
    with nlp_doc.retokenize() as retokenizer:
        for ent in nlp_doc.ents:
            retokenizer.merge(ent)
    tokens = map(replace_person_names, nlp_doc)
    return "".join(tokens)

survey_doc = nlp(survey_text)
print(redact_names(survey_doc))