10 examples of 'remove stop words from string python' in Python

73
def remove_stopwords(tokens, language):
74
    """
75
    去除中文结束词
76
    """
77
    from .stopwords import stopwords
78
79
    # 删除结束词
80
    tokens = set(tokens) - set(stopwords)
81
82
    return tokens

53
def remove_stopwords(tokens, stopwords=STOPWORDS):
54
    """Remove stopwords using list from `gensim.parsing.preprocessing.STOPWORDS`.
55
56
    Parameters
57
    ----------
58
    tokens : iterable of str
59
        Sequence of tokens.
60
    stopwords : iterable of str, optional
61
        Sequence of stopwords
62
63
    Returns
64
    -------
65
    list of str
66
        List of tokens without `stopwords`.
67
68
    """
69
    return [token for token in tokens if token not in stopwords]

231
def _remove_stop_words(phrase):
232
    while len(phrase) > 0 and (phrase[0].is_stop
233
                               or str(phrase[0]).strip().lower() in Stopwords.get_words()):
234
        phrase = phrase[1:]
235
    while len(phrase) > 0 and (phrase[-1].is_stop
236
                               or str(phrase[-1]).strip().lower() in Stopwords.get_words()):
237
        phrase = phrase[:-1]
238
    return phrase

6
def stopwords_filter(string):
7
8
    text = string
9
    # strip tashkeel because the stop words list contains voweled words
10
    text = araby.strip_tashkeel(text)
11
    word_tokenizer = WordTokenizer("arabic")
12
    tokens = word_tokenizer.tokenize(text)
13
14
    # filter stop words
15
    no_stops = [w for w in tokens if w not in ARABIC_STOPS]
16
17
    return no_stops

35
@staticmethod
36
def removeStopWords(words):
37
    file = open("stopwords.dic")
38
    stopwords = []
39
    for line in file.readlines():
40
        line = line.strip();
41
        if len(line) > 0:
42
            stopwords.append(line)
43
    file.close()
44
    rwords = []
45
    for word in words:
46
        flag = True
47
        for stopword in stopwords:
48
            #if word.encode('utf-8') == stopword.encode('utf-8'):
49
            if word == stopword:
50
                flag = False
51
                break
52
        if flag and len(word.strip()) > 0:
53
            rwords.append(word)
54
    return rwords

433
def stop_words_stem(self, stop_words=None):
434
    if stop_words is not None:
435
        stop_words_ = stop_words
436
    else:
437
        stop_words_ = self.stop_words
438
    return list(set([stem(word) for word in stop_words_]))

478
def split_stop_keep_word_string(input_string: str) -> List[str]:
479
    """Breaks stop and keepword string inputs into lists of words.
480
481
    :param input_string: A string of words input by the user.
482
    :return: A list of the user's string broken up into words.
483
    """
484
485
    input_lines = input_string.split("\n")
486
487
    # A list of all words delimited by commas, spaces, and newlines
488
    input_words = [word
489
                   for line in input_lines
490
                   for word in re.split('[, ]', line.strip())
491
                   if word != '']
492
493
    return input_words

44
def get_stop_words(file_path='./data/stop_words.txt'):
45
    """获取停用词列表"""
46
    stop_words_set = set()
47
    with open(file_path, 'rb') as f:
48
        for word in f:
49
            if not word.startswith('//'):
50
                stop_words_set.add(word.strip().decode('utf8'))
51
    # 处理空字符串
52
    if '' in stop_words_set:
53
        stop_words_set.remove('')
54
55
    return stop_words_set

30
def parse_stopwords(handle):
31
    """Parse a file with stopwords in it into a list of stopwords.
32
33
    :param handle: an readable file handle.
34
    :returns: An iterator that will yield stopwords.
35
    """
36
    for line in handle:
37
        yield line.strip().lower()

188
def stop(tokensin, unigrams, ngrams, digits=True, tuples=False):
189
    new_tokens = []
190
    for element in tokensin:
191
        token = element[1] if tuples else element
192
        ngram = token.split()
193
        if any([x.isdigit() for x in ngram]) and digits:
194
            continue
195
        if len(ngram) == 1:
196
            if ngram[0] not in unigrams:
197
                new_tokens.append(element)
198
        else:
199
            if token not in ngrams:
200
                new_tokens.append(element)
201
    return new_tokens