-
Notifications
You must be signed in to change notification settings - Fork 0
/
ckd_swar_keyword.R
184 lines (147 loc) · 6.04 KB
/
ckd_swar_keyword.R
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
#3. Conduct keyword co-occurence and cluster analysis
#Load libraries
library(litsearchr)
library(igraph)
library(wordcloud)
library(RColorBrewer)
library(tidyverse)
#Create a one-column dataframe of combined titles and abstracts
#This uses the dataframe produced at the end of ckd_swar_dboverlap.R
unique_tiab <- unique_all_bib[, c("duplicate_id", "title","abstract")]
unique_tiab$combined <- paste(unique_tiab$title, unique_tiab$abstract, sep = " ")
unique_tiab <- unique_tiab[, c("duplicate_id","combined")]
unique_tiab <- as.data.frame(unique_tiab)
#Use litsearchr to extract terms and create keyword co-occurence network
#extract terms that are bigrams and occur at least three times
#Can play around with these settings
extracted_terms_pass1 <- litsearchr::extract_terms(text=unique_tiab[, "combined"],
method="fakerake",
min_freq=3,
min_n=2)
#Write out extracted_terms for review and mark non-relevant terms
#Save text file of non-relevant terms to remove_bigrams.txt
writeLines(extracted_terms_pass1, "outputs/extracted_terms_pass1.txt")
#Use this additional list of stopwords from Luke Tudge tutorial:
#https://luketudge.github.io/litsearchr-tutorial/litsearchr_tutorial.html
clinpsy_stopwords <- read_lines("data/clin_psy_stopwords.txt")
#Create final list of stopwords
all_stopwords <- c(get_stopwords("English"), clinpsy_stopwords)
#Extract final list of terms for network
extracted_terms <- extract_terms(
text=unique_tiab[, "combined"],
method="fakerake",
min_freq=3, min_n=2,
stopwords=all_stopwords)
#Use manually identified non-relevant terms from above and remove from extracted terms
bigrams_to_remove <- readLines("data/remove_bigrams.txt")
cleaned_terms <- setdiff(extracted_terms, bigrams_to_remove)
#Create object of terms from title/abstract in unique_tiab
docs <- paste(unique_tiab[, "combined"])
#Create matrix of the terms of interest (i.e. extracted_terms) based on docs
dfm <- create_dfm(elements=docs, features=cleaned_terms)
#Create graph object only including terms that appear in at least 3 records
#Can play around with this setting
g <- create_network(dfm, min_studies=3)
#Remove nodes with less than 5 edges
isolated <- which(degree(g)==5)
g2 <- delete.vertices(g, isolated)
#Set layout
layout_fr = layout_with_fr(g2)
#Plot network
plot(g2, edge.color = "lightgray", edge.width = 0.2, vertex.size = 2, vertex.label=NA, layout = layout_fr)
#Identify communities
my_clusters_1 <- cluster_louvain(g2, weights = NULL)
#Print number of communities and size of each
length(my_clusters_1)
sizes(my_clusters_1)
## Identify which communities have fewer than 20 nodes (play around with this parameter)
small <- which(table(my_clusters_1$membership) < 20)
## Keep only nodes in communities with greater than 20 members (play around with this)
keep <- V(g2)[!(my_clusters_1$membership %in% small)]
#Make graph object removing small communities based on g2 clusters
g3 <- induced_subgraph(g2, keep)
#Redo communities and layout
my_clusters_2 <- cluster_louvain(g3)
layout_fr_2 <- layout_fr[keep,]
length(my_clusters_2)
sizes(my_clusters_2)
#Set colors for nodes
color_palette <- c("#7FC97F", "#BEAED4", "#FDC086", "#FFFF99", "#A6CEE3", "#E78AC3")
i <- V(g3)$name %in% names(membership(my_clusters_2))
V(g3)$color[i] <- color_palette[membership(my_clusters_2)]
#Plot network
pdf("plots/network_graph.pdf", width = 8, height = 6) # Adjust width, height, and res as needed
plot(g3,
vertex.label = NA,
vertex.size=3,
edge.color = "lightgray",
edge.width = 0.2,
layout=layout_fr_2,
edge.curved = TRUE,
mark.groups=NULL)
dev.off()
#Assign cluster groups to nodes
#cluster_grps <- membership(my_clusters_2)
cluster_grps_names <- cbind(V(g3)$name, my_clusters_2$membership)
#Count the number of times extracted terms occur in the whole dataset
phrase_counts <- sapply(extracted_terms, function(phrase) sum(str_count(docs, phrase)))
result_df <- data.frame(Phrase = names(phrase_counts), Count = as.vector(phrase_counts))
#Merge to cluster dataframe
word_freq_cluster <- merge(cluster_grps_names, result_df, by.x = "V1", by.y = "Phrase", all.x = TRUE)
word_freq_cluster_sub <- subset(word_freq_cluster, Count != 0)
write.csv(word_freq_cluster_sub, "outputs/word_freq_cluster.csv")
#Word cloud
png(filename="plots/wc1.png")
wc1 <- word_freq_cluster_sub %>%
filter(V2 == 1) %>%
with(wordcloud(V1, Count, min.freq = 1,
#max.words=100,
random.order=FALSE,
rot.per=0,
colors = brewer.pal(8, "Dark2")))
dev.off()
png(filename="plots/wc2.png")
wc2 <- word_freq_cluster_sub %>%
filter(V2 == 2) %>%
with(wordcloud(V1, Count, min.freq = 1,
#max.words=100,
random.order=FALSE,
rot.per=0,
colors = brewer.pal(8, "Dark2")))
dev.off()
png(filename="plots/wc3.png")
wc3 <- word_freq_cluster_sub %>%
filter(V2 == 3) %>%
with(wordcloud(V1, Count, min.freq = 1,
#max.words=100,
random.order=FALSE,
rot.per=0,
colors = brewer.pal(8, "Dark2")))
dev.off()
png(filename="plots/wc4.png")
wc4 <- word_freq_cluster_sub %>%
filter(V2 == 4) %>%
with(wordcloud(V1, Count, min.freq = 1,
#max.words=100,
random.order=FALSE,
rot.per=0,
colors = brewer.pal(8, "Dark2")))
dev.off()
png(filename="plots/wc5.png")
wc5 <- word_freq_cluster_sub %>%
filter(V2 == 5) %>%
with(wordcloud(V1, Count, min.freq = 1,
#max.words=100,
random.order=FALSE,
rot.per=0,
colors = brewer.pal(8, "Dark2")))
dev.off()
png(filename="plots/wc6.png")
wc6 <- word_freq_cluster_sub %>%
filter(V2 == 6) %>%
with(wordcloud(V1, Count, min.freq = 1,
#max.words=100,
random.order=FALSE,
rot.per=0,
colors = brewer.pal(8, "Dark2")))
dev.off()