Wsrtjtyk

I am trying to clean up my data to remove; i.) special characters (e.g
+_), ii.) specific words (e.g retweet, followers, couldn, better, person) iii.) words that do not appear in the english dictionary I am using the quanteda library. My objective is to get the top 50 bigrams and plot them on a graph.

install.packages("textcat")

library(tm)

library(textcat)

the_data <- read.csv("twitterData.csv")

tweets_data <- the_data$x

tweets_corpus <- Corpus(VectorSource(tweets_data))

subSpace <- content_transformer(function(x, pattern) gsub(pattern, 

                                                          " ", x))

twitterHandleRemover <- function(x) gsub("@\S+","", x)

shortWordRemover <- function(x) gsub('\b\w{1,5}\b','',x)

urlRemover <- function(x) gsub("http:[[:alnum:]]*","", x)

hashtagRemover <- function(x) gsub("#\S+","", x)

tweets_corpus <- tm_map(tweets_corpus, subSpace, "/")

tweets_corpus <- tm_map(tweets_corpus, subSpace, "@")

tweets_corpus <- tm_map(tweets_corpus, subSpace, "\|%&*#+_><")

tweets_corpus <- tm_map(tweets_corpus, content_transformer(tolower))

tweets_corpus <- tm_map(tweets_corpus, removeNumbers)

tweets_corpus <- tm_map(tweets_corpus, content_transformer(urlRemover))

tweets_corpus <- tm_map(tweets_corpus, 

                        content_transformer(shortWordRemover))

tweets_corpus <- tm_map(tweets_corpus, 

                        content_transformer(twitterHandleRemover))

tweets_corpus <- tm_map(tweets_corpus, 

                        content_transformer(hashtagRemover))

tweets_corp<- corpus(tweets_corpus)



tweets_dfm <- tokens(tweets_corp, remove_numbers = T, 

                     remove_hyphens = T) %>%

   tokens_remove("\p{P}", valuetype = "regex", padding=TRUE) %>%

   tokens_remove(stopwords("english"), padding=TRUE) %>%

   tokens_remove("\d+", padding = TRUE) %>%

   tokens_ngrams(n=2) %>% dfm()



topfeatures(tweets_dfm,50)

This is output from my code:

Edit

I have tried using

specialChars <- function(x) gsub("[^[:alnum:]///']","", x)

tweets_corpus <- tm_map(tweets_corpus, 

content_transformer(specialChars))

to remove special characters, but that seems to remove all characters - output is numeric(0)

Why not just do something like this:

> x <- "je n'aime pas ça"

> Encoding(x)

[1] "latin1"

> iconv(x, from = "latin1", to = "ASCII//TRANSLIT")

[1] "je n'aime pas ca"

So do iconv(tweets_data, from = "latin1", to = "ASCII//TRANSLIT") assuming your data is in latin1

And next keep only alphanumeric characters or spaces

gsub(pattern = "[^[:alnum:][:space:]]", " ", "<friends @symbols")