i am having trouble with tweets.

I copied the get_user_timeline function from helper to tweets, since it was origanily set to def get_user_timeline(screen_name, count=200):

I am not clear why anyone would set a counter as fixed in a function definition? Why not put it in the body of the function if you want to set it to a hard value? Or as a magic number line in C. #defind counter 200. My guess is the function take one value not two.

I am getting an error, i do not completely understand. I get that "TypeError: expected string or bytes-like object" but text in my code should be a string. The error is happen inside the tokenizer.

my code is:

#!/usr/bin/env python3

import html
import os
import plotly
import socket
import nltk
import sys

from twython import Twython
from twython import TwythonAuthError, TwythonError, TwythonRateLimitError
from analyzer import Analyzer
from termcolor import colored
from nltk.tokenize import TweetTokenizer

def main():
    # ensure proper usage
    if len(sys.argv) != 2:
        sys.exit("Usage: ./tweets @name")

    twe = get_user_timeline(sys.argv[1])
    if twe == None:
        sys.exit("Failed to retrieve tweets")    

        # absolute paths to lists
    positives = os.path.join(sys.path[0], "positive-words.txt")
    negatives = os.path.join(sys.path[0], "negative-words.txt")

    # instantiate analyzer
    analyzer = Analyzer(positives, negatives)

    tokenizer = nltk.tokenize.TweetTokenizer()

    for tweet in twe:
        token = tokenizer.tokenize(tweet)
        score = analyzer.analyze(token)
        if score > 0.0:
            print(colored(tweet, "green"))
        elif score < 0.0:
            print(colored(tweet, "red"))
            print(colored("tweet", "yellow"))

def get_user_timeline(screen_name, count=50):
    """Return list of most recent tweets posted by screen_name."""

    # ensure count is valid
    if count < 1 or count > 200:
        raise RuntimeError("invalid count")

    # ensure environment variables are set
    if not os.environ.get("API_KEY"):
        raise RuntimeError("API_KEY not set")
    if not os.environ.get("API_SECRET"):
        raise RuntimeError("API_SECRET not set")

    # get screen_name's most recent tweets
    # https://dev.twitter.com/rest/reference/get/users/lookup
    # https://dev.twitter.com/rest/reference/get/statuses/user_timeline
    # https://github.com/ryanmcgrath/twython/blob/master/twython/endpoints.py
        twitter = Twython(os.environ.get("API_KEY"), os.environ.get("API_SECRET"))
        user = twitter.lookup_user(screen_name=screen_name)
        if user[0]["protected"]:
            return None
        tweets = twitter.get_user_timeline(screen_name=screen_name, count=count)
        return [html.unescape(tweet["text"].replace("\n", " ")) for tweet in tweets]
    except TwythonAuthError:
        raise RuntimeError("invalid API_KEY and/or API_SECRET") from None
    except TwythonRateLimitError:
        raise RuntimeError("you've hit a rate limit") from None
    except TwythonError:
        return None

if __name__ == "__main__":

The error i am getting is:

./tweets katyperry
Traceback (most recent call last):
  File "./tweets", line 81, in <module>
  File "./tweets", line 39, in main
    score = analyzer.analyze(token)
  File "/home/m/sentiments/analyzer.py", line 31, in analyze
    token = word_tokenize(text)
  File "/home/m/.local/lib/python3.5/site-packages/nltk/tokenize/__init__.py", line 109, in word_tokenize
    return [token for sent in sent_tokenize(text, language)
  File "/home/m/.local/lib/python3.5/site-packages/nltk/tokenize/__init__.py", line 94, in sent_tokenize
    return tokenizer.tokenize(text)
  File "/home/m/.local/lib/python3.5/site-packages/nltk/tokenize/punkt.py", line 1237, in tokenize
    return list(self.sentences_from_text(text, realign_boundaries))
  File "/home/m/.local/lib/python3.5/site-packages/nltk/tokenize/punkt.py", line 1285, in sentences_from_text
    return [text[s:e] for s, e in self.span_tokenize(text, realign_boundaries)]
  File "/home/m/.local/lib/python3.5/site-packages/nltk/tokenize/punkt.py", line 1276, in span_tokenize
    return [(sl.start, sl.stop) for sl in slices]
  File "/home/m/.local/lib/python3.5/site-packages/nltk/tokenize/punkt.py", line 1276, in <listcomp>
    return [(sl.start, sl.stop) for sl in slices]
  File "/home/m/.local/lib/python3.5/site-packages/nltk/tokenize/punkt.py", line 1316, in _realign_boundaries
    for sl1, sl2 in _pair_iter(slices):
  File "/home/m/.local/lib/python3.5/site-packages/nltk/tokenize/punkt.py", line 310, in _pair_iter
    prev = next(it)
  File "/home/m/.local/lib/python3.5/site-packages/nltk/tokenize/punkt.py", line 1289, in _slices_from_text
    for match in self._lang_vars.period_context_re().finditer(text):
TypeError: expected string or bytes-like object

The error says i need a string, the error is inside nltk. token should be a text string from the tweet, what what i am sending to nltk.

Don't think the problem is in analyzer, but my code for analyzer is:

import nltk

from nltk.tokenize import word_tokenize

class Analyzer():
    """Implements sentiment analysis."""

    def __init__(self, positives, negatives):
        """Initialize Analyzer."""
        self.pos = set()
        self.neg = set()

        pos1 = open(positives, "r")
        for line in pos1:
            if not li.startswith(';'):

        neg1 = open(negatives, "r")
        for line in neg1:
            if not li.startswith(';'):

    def analyze(self, text):
        """Analyze text for sentiment, returning its score."""
        score = 0
        token = word_tokenize(text)
        for word in token:
            if word.lower() in self.pos:
                score += 1
            elif word.lower() in self.neg:
                score -= 1
        return score

1 Answer 1


This definition def get_user_timeline(screen_name, count=200): means the function takes one argument or two. If the call is get_user_timeline(katyperry) it will return 200 tweets. If the call is get_user_timeline(katyperry,50) it will return 50 tweets. 200 is the default if count argument is not supplied. So no need to recreate the function, just send the desired count.

You are not sending a string to word_tokenize because tokenizer.tokenize() returns a list.

  • Thanks, i fixed it.
    – Manny_Mar
    May 11, 2017 at 2:49
  • I did not understand fcn(x, y=20). I looked at the week 8 notes, and did not see it there. I do not remember it from the video. i'm using helper now, instead of pasting the function in tweet.
    – Manny_Mar
    May 11, 2017 at 3:00
  • I was surprised that the syntax wasn't commented on or even hinted in any of the materials. May 11, 2017 at 10:45

You must log in to answer this question.

Not the answer you're looking for? Browse other questions tagged .