mirror of
https://github.com/pds-nest/nest.git
synced 2024-11-22 04:54:18 +00:00
✨ Implement analysis visualization
This commit is contained in:
parent
b58e49b5b3
commit
b471fc17c0
21 changed files with 294 additions and 108 deletions
|
@ -46,7 +46,7 @@ export default {
|
|||
repoEdit: "Modifica repository",
|
||||
menuActive: "Le tue repository attive",
|
||||
menuArchived: "Le tue repository archiviate",
|
||||
emptyMenu: "Non c'è nulla qui",
|
||||
emptyMenu: "Non c'è nulla qui.",
|
||||
delete: "Elimina",
|
||||
archive: "Archivia",
|
||||
edit: "Modifica",
|
||||
|
@ -117,7 +117,7 @@ export default {
|
|||
repoEdit: "Edit repository",
|
||||
menuActive: "Your active repositories",
|
||||
menuArchived: "Your archived repositories",
|
||||
emptyMenu: "There's nothing here",
|
||||
emptyMenu: "There's nothing here.",
|
||||
delete: "Delete",
|
||||
archive: "Archive",
|
||||
edit: "Edit",
|
||||
|
@ -188,7 +188,7 @@ export default {
|
|||
repoEdit: "Muokkaa arkistoa",
|
||||
menuActive: "Aktiiviset arkistosi",
|
||||
menuArchived: "Arkistoidut arkistosi",
|
||||
emptyMenu: "Täällä ei ole mitään",
|
||||
emptyMenu: "Täällä ei ole mitään.",
|
||||
delete: "Poista",
|
||||
archive: "Arkistoi",
|
||||
edit: "Muokkaa",
|
||||
|
|
|
@ -4,7 +4,7 @@ import BoxFull from "./BoxFull"
|
|||
import { MapContainer, TileLayer } from "react-leaflet"
|
||||
|
||||
|
||||
export default function BoxMap({ header, setMap, startingPosition, startingZoom, button, children, additions, ...props }) {
|
||||
export default function BoxMap({ header, setMap, startingPosition = { lat: 41.89309, lng: 12.48289 }, startingZoom = 3, button, children, ...props }) {
|
||||
return (
|
||||
<BoxFull
|
||||
header={header}
|
||||
|
@ -21,7 +21,7 @@ export default function BoxMap({ header, setMap, startingPosition, startingZoom,
|
|||
attribution='(c) <a href="https://osm.org/copyright">OpenStreetMap contributors</a>'
|
||||
url="https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png"
|
||||
/>
|
||||
{additions}
|
||||
{children}
|
||||
<div className={"leaflet-top leaflet-right"}>
|
||||
<div className={"leaflet-control"}>
|
||||
{button}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
import React, { useContext } from "react"
|
||||
import React from "react"
|
||||
import BoxFull from "../base/BoxFull"
|
||||
import ReactWordcloud from "@steffo/nest-react-wordcloud"
|
||||
import ContextLanguage from "../../contexts/ContextLanguage"
|
||||
import Style from "./BoxWordcloud.module.css"
|
||||
|
||||
|
||||
/**
|
||||
|
@ -12,19 +12,17 @@ import ContextLanguage from "../../contexts/ContextLanguage"
|
|||
* @returns {JSX.Element}
|
||||
* @constructor
|
||||
*/
|
||||
export default function BoxWordcloud({ words, props }) {
|
||||
const { strings } = useContext(ContextLanguage)
|
||||
|
||||
export default function BoxWordcloud({ words, ...props }) {
|
||||
return (
|
||||
<BoxFull header={strings.wordcloud} {...props}>
|
||||
<div style={{ "width": "100%", "height": "100%" }}>
|
||||
<BoxFull {...props}>
|
||||
<div className={Style.WordcloudContainer}>
|
||||
<ReactWordcloud
|
||||
options={{
|
||||
colors: [
|
||||
"var(--fg-primary)",
|
||||
],
|
||||
fontFamily: "Bree Serif",
|
||||
fontSizes: [16, 48],
|
||||
fontSizes: [8, 64],
|
||||
size: undefined,
|
||||
deterministic: true,
|
||||
}}
|
4
nest_frontend/components/base/BoxWordcloud.module.css
Normal file
4
nest_frontend/components/base/BoxWordcloud.module.css
Normal file
|
@ -0,0 +1,4 @@
|
|||
.WordcloudContainer {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
|
@ -1,6 +1,6 @@
|
|||
.SummaryLeft {
|
||||
min-width: 175px;
|
||||
width: 275px;
|
||||
width: 260px;
|
||||
flex-shrink: 0;
|
||||
|
||||
display: grid;
|
||||
grid-template-areas:
|
||||
|
|
|
@ -8,9 +8,6 @@ import ContextLanguage from "../../contexts/ContextLanguage"
|
|||
import BoxMap from "../base/BoxMap"
|
||||
|
||||
|
||||
const STARTING_POSITION = { lat: 41.89309, lng: 12.48289 }
|
||||
const STARTING_ZOOM = 3
|
||||
|
||||
/**
|
||||
* https://wiki.openstreetmap.org/wiki/Zoom_levels
|
||||
*/
|
||||
|
@ -47,8 +44,8 @@ const MPIXEL = [
|
|||
* @constructor
|
||||
*/
|
||||
export default function BoxConditionMap({ ...props }) {
|
||||
const [position, setPosition] = useState(STARTING_POSITION)
|
||||
const [zoom, setZoom] = useState(STARTING_ZOOM)
|
||||
const [position, setPosition] = useState()
|
||||
const [zoom, setZoom] = useState()
|
||||
const [map, setMap] = useState(null)
|
||||
const { addCondition } = useRepositoryEditor()
|
||||
const { strings } = useContext(ContextLanguage)
|
||||
|
@ -92,7 +89,6 @@ export default function BoxConditionMap({ ...props }) {
|
|||
"COORDINATES",
|
||||
`< ${radius} ${position.lat} ${position.lng}`,
|
||||
))
|
||||
setPosition(STARTING_POSITION)
|
||||
}
|
||||
|
||||
return (
|
||||
|
@ -106,8 +102,6 @@ export default function BoxConditionMap({ ...props }) {
|
|||
{strings.byZone}
|
||||
</span>
|
||||
}
|
||||
startingPosition={STARTING_POSITION}
|
||||
startingZoom={STARTING_ZOOM}
|
||||
setMap={setMap}
|
||||
button={
|
||||
<ButtonIconOnly
|
||||
|
|
|
@ -3,6 +3,7 @@ import BoxFullScrollable from "../base/BoxFullScrollable"
|
|||
import Loading from "../base/Loading"
|
||||
import ContextLanguage from "../../contexts/ContextLanguage"
|
||||
import SummaryRepository from "./SummaryRepository"
|
||||
import Empty from "./Empty"
|
||||
|
||||
|
||||
/**
|
||||
|
@ -21,14 +22,12 @@ import SummaryRepository from "./SummaryRepository"
|
|||
* @constructor
|
||||
*/
|
||||
export default function BoxRepositories({ repositories, view, archive, edit, destroy, loading, running, className, ...props }) {
|
||||
const { strings } = useContext(ContextLanguage)
|
||||
|
||||
let contents
|
||||
if(loading) {
|
||||
contents = <Loading/>
|
||||
}
|
||||
else if(repositories.length === 0) {
|
||||
contents = <i>{strings.emptyMenu}</i>
|
||||
contents = <Empty/>
|
||||
}
|
||||
else {
|
||||
contents = repositories.map(repo => (
|
||||
|
|
|
@ -1,14 +1,23 @@
|
|||
import React from "react"
|
||||
import BoxFullScrollable from "../base/BoxFullScrollable"
|
||||
import SummaryTweet from "./SummaryTweet"
|
||||
import Empty from "./Empty"
|
||||
|
||||
|
||||
export default function BoxRepositoryTweets({ tweets, ...props }) {
|
||||
// TODO: Translate this
|
||||
|
||||
let content
|
||||
if(tweets.length === 0) {
|
||||
content = <Empty/>
|
||||
}
|
||||
else {
|
||||
content = tweets.map(tweet => <SummaryTweet key={tweet["snowflake"]} tweet={tweet}/>)
|
||||
}
|
||||
|
||||
return (
|
||||
<BoxFullScrollable header={"Tweets"} {...props}>
|
||||
{tweets.map(tweet => <SummaryTweet key={tweet.snowflake} tweet={tweet}/>)}
|
||||
{content}
|
||||
</BoxFullScrollable>
|
||||
)
|
||||
}
|
||||
|
|
|
@ -0,0 +1,14 @@
|
|||
import React from "react"
|
||||
import BoxFull from "../base/BoxFull"
|
||||
|
||||
|
||||
export default function BoxVisualizationGraph({ children, className, ...props }) {
|
||||
// TODO: translate this
|
||||
// TODO: implement this
|
||||
|
||||
return (
|
||||
<BoxFull header={"Hourly graph"} {...props}>
|
||||
{children}
|
||||
</BoxFull>
|
||||
)
|
||||
}
|
40
nest_frontend/components/interactive/BoxVisualizationMap.js
Normal file
40
nest_frontend/components/interactive/BoxVisualizationMap.js
Normal file
|
@ -0,0 +1,40 @@
|
|||
import React, { useContext } from "react"
|
||||
import BoxMap from "../base/BoxMap"
|
||||
import ContextLanguage from "../../contexts/ContextLanguage"
|
||||
import { Marker, Popup } from "react-leaflet"
|
||||
|
||||
|
||||
const locationRegex = /[{](?<lat>[0-9.]+),(?<lng>[0-9.]+)[}]/
|
||||
|
||||
export default function BoxVisualizationMap({ tweets, ...props }) {
|
||||
// TODO: translate this
|
||||
const {strings} = useContext(ContextLanguage)
|
||||
|
||||
console.debug(tweets)
|
||||
const markers = tweets.filter(tweet => tweet.location).map(tweet => {
|
||||
const match = locationRegex.exec(tweet.location)
|
||||
if(!match) {
|
||||
console.error("No match for location ", tweet.location)
|
||||
return null
|
||||
}
|
||||
const {lat, lng} = match.groups
|
||||
return (
|
||||
<Marker key={tweet["snowflake"]} position={[Number.parseFloat(lat), Number.parseFloat(lng)]}>
|
||||
<Popup>
|
||||
<p>
|
||||
{tweet["content"]}
|
||||
</p>
|
||||
<p>
|
||||
— <a href={`https://twitter.com/${tweet["poster"]}/status/${tweet["snowflake"]}`}>@{tweet["poster"]}</a>
|
||||
</p>
|
||||
</Popup>
|
||||
</Marker>
|
||||
)
|
||||
})
|
||||
|
||||
return (
|
||||
<BoxMap header={"Map"} {...props}>
|
||||
{markers}
|
||||
</BoxMap>
|
||||
)
|
||||
}
|
|
@ -0,0 +1,77 @@
|
|||
import React, { useMemo } from "react"
|
||||
import FormLabelled from "../base/FormLabelled"
|
||||
import FormLabel from "../base/formparts/FormLabel"
|
||||
import BoxFullScrollable from "../base/BoxFullScrollable"
|
||||
import tokenizeTweetWords from "../../utils/tokenizeTweetWords"
|
||||
|
||||
|
||||
export default function BoxVisualizationStats({ tweets, totalTweetCount, ...props }) {
|
||||
|
||||
const words = useMemo(
|
||||
() => tokenizeTweetWords(tweets),
|
||||
[tweets]
|
||||
)
|
||||
|
||||
const tweetCount = tweets.length
|
||||
const tweetPct = tweetCount / totalTweetCount * 100
|
||||
const tweetLocationCount = tweets.filter(tweet => tweet.location).length
|
||||
const tweetLocationPct = tweetLocationCount / tweetCount * 100
|
||||
const tweetContent = tweets.filter(tweet => tweet.content)
|
||||
const tweetContentCount = tweetContent.length
|
||||
const tweetContentPct = tweetContentCount / tweetCount * 100
|
||||
const wordCount = words.map(word => word.value).reduce((a, b) => a+b)
|
||||
const mostPopularWord = words.sort((wa, wb) => {
|
||||
if(wa.value > wb.value) return -1
|
||||
if(wa.value < wb.value) return 1
|
||||
return 0
|
||||
})[0].text
|
||||
const users = [...new Set(tweets.map(tweet => tweet.poster))]
|
||||
const usersCount = users.length
|
||||
|
||||
// TODO: tweets with picture count
|
||||
// TODO: tweets with picture pct
|
||||
|
||||
// TODO: translate this
|
||||
return (
|
||||
<BoxFullScrollable header={"Stats"} {...props}>
|
||||
<FormLabelled>
|
||||
<FormLabel text={"Total tweets"}>
|
||||
<b>{totalTweetCount}</b>
|
||||
</FormLabel>
|
||||
<FormLabel text={"Displayed tweets"}>
|
||||
<b>{tweetCount}</b>
|
||||
</FormLabel>
|
||||
<FormLabel text={"% of displayed tweets"}>
|
||||
<b>{tweetPct.toFixed(2)}%</b>
|
||||
</FormLabel>
|
||||
<FormLabel text={"Tweets with location"}>
|
||||
<b>{tweetLocationCount}</b>
|
||||
</FormLabel>
|
||||
<FormLabel text={"% of tweets with location"}>
|
||||
<b>{tweetLocationPct.toFixed(2)}%</b>
|
||||
</FormLabel>
|
||||
<FormLabel text={"Tweets with content"}>
|
||||
<b>{tweetContentCount}</b>
|
||||
</FormLabel>
|
||||
<FormLabel text={"% of tweets with content"}>
|
||||
<b>{tweetContentPct.toFixed(2)}%</b>
|
||||
</FormLabel>
|
||||
<FormLabel text={"Word count"}>
|
||||
<b>{wordCount}</b>
|
||||
</FormLabel>
|
||||
<FormLabel text={"Most popular word"}>
|
||||
<b>{mostPopularWord}</b>
|
||||
</FormLabel>
|
||||
<FormLabel text={"Tweets with image"}>
|
||||
<b>🚧</b>
|
||||
</FormLabel>
|
||||
<FormLabel text={"% of tweets with image"}>
|
||||
<b>🚧</b>
|
||||
</FormLabel>
|
||||
<FormLabel text={"Users count"}>
|
||||
<b>{usersCount}</b>
|
||||
</FormLabel>
|
||||
</FormLabelled>
|
||||
</BoxFullScrollable>
|
||||
)
|
||||
}
|
|
@ -0,0 +1,18 @@
|
|||
import React, { useContext, useMemo } from "react"
|
||||
import BoxWordcloud from "../base/BoxWordcloud"
|
||||
import ContextLanguage from "../../contexts/ContextLanguage"
|
||||
import tokenizeTweetWords from "../../utils/tokenizeTweetWords"
|
||||
|
||||
|
||||
export default function BoxVisualizationWordcloud({ tweets = [], ...props }) {
|
||||
const {strings} = useContext(ContextLanguage)
|
||||
|
||||
const words = useMemo(
|
||||
() => tokenizeTweetWords(tweets),
|
||||
[tweets]
|
||||
)
|
||||
|
||||
return (
|
||||
<BoxWordcloud header={strings.wordcloud} words={words} {...props}/>
|
||||
)
|
||||
}
|
15
nest_frontend/components/interactive/Empty.js
Normal file
15
nest_frontend/components/interactive/Empty.js
Normal file
|
@ -0,0 +1,15 @@
|
|||
import React, { useContext } from "react"
|
||||
import Style from "./Empty.module.css"
|
||||
import classNames from "classnames"
|
||||
import ContextLanguage from "../../contexts/ContextLanguage"
|
||||
|
||||
|
||||
export default function Empty({ children, className, ...props }) {
|
||||
const { strings } = useContext(ContextLanguage)
|
||||
|
||||
return (
|
||||
<i className={classNames(Style.Empty, className)} {...props}>
|
||||
{strings.emptyMenu}
|
||||
</i>
|
||||
)
|
||||
}
|
3
nest_frontend/components/interactive/Empty.module.css
Normal file
3
nest_frontend/components/interactive/Empty.module.css
Normal file
|
@ -0,0 +1,3 @@
|
|||
.Empty {
|
||||
opacity: 0.5;
|
||||
}
|
|
@ -1,15 +1,18 @@
|
|||
import React from "react"
|
||||
import SummaryBase from "../base/summary/SummaryBase"
|
||||
import SummaryLeft from "../base/summary/SummaryLeft"
|
||||
import { faComment, faMapPin } from "@fortawesome/free-solid-svg-icons"
|
||||
import { faComment, faLocationArrow, faMapMarker, faMapMarkerAlt, faMapPin } from "@fortawesome/free-solid-svg-icons"
|
||||
import SummaryText from "../base/summary/SummaryText"
|
||||
import SummaryRight from "../base/summary/SummaryRight"
|
||||
|
||||
|
||||
export default function SummaryTweet({ tweet, ...props }) {
|
||||
let icon
|
||||
if(tweet.place) {
|
||||
icon = faMapPin
|
||||
if(tweet["location"]) {
|
||||
icon = faMapMarkerAlt
|
||||
}
|
||||
else if(tweet["place"]) {
|
||||
icon = faLocationArrow
|
||||
}
|
||||
else {
|
||||
icon = faComment
|
||||
|
@ -19,9 +22,9 @@ export default function SummaryTweet({ tweet, ...props }) {
|
|||
<SummaryBase {...props}>
|
||||
<SummaryLeft
|
||||
icon={icon}
|
||||
title={`@${tweet.poster}`}
|
||||
subtitle={tweet.place}
|
||||
onClick={() => window.open(`https://twitter.com/${tweet.poster}/status/${tweet.snowflake}`)}
|
||||
title={`@${tweet["poster"]}`}
|
||||
subtitle={new Date(tweet["insert_time"]).toLocaleString()}
|
||||
onClick={() => window.open(`https://twitter.com/${tweet["poster"]}/status/${tweet["snowflake"]}`)}
|
||||
/>
|
||||
<SummaryText>
|
||||
{tweet.content}
|
||||
|
|
|
@ -1,22 +0,0 @@
|
|||
import React from "react"
|
||||
import Style from "./RepositoryViewer.module.css"
|
||||
import classNames from "classnames"
|
||||
import ContextRepositoryViewer from "../../contexts/ContextRepositoryViewer"
|
||||
|
||||
|
||||
export default function RepositoryViewer({
|
||||
id,
|
||||
}) {
|
||||
return (
|
||||
<ContextRepositoryViewer.Provider
|
||||
value={{
|
||||
id,
|
||||
|
||||
}}
|
||||
>
|
||||
<div className={classNames(Style.RepositoryViewer, className)}>
|
||||
|
||||
</div>
|
||||
</ContextRepositoryViewer.Provider>
|
||||
)
|
||||
}
|
|
@ -1,14 +0,0 @@
|
|||
import { useContext } from "react"
|
||||
import ContextRepositoryViewer from "../contexts/ContextRepositoryViewer"
|
||||
|
||||
|
||||
/**
|
||||
* Hook to quickly use {@link ContextRepositoryViewer}.
|
||||
*/
|
||||
export default function useRepositoryViewer() {
|
||||
const context = useContext(ContextRepositoryViewer)
|
||||
if(!context) {
|
||||
throw new Error("This component must be placed inside a RepositoryViewer.")
|
||||
}
|
||||
return context
|
||||
}
|
|
@ -1,8 +1,7 @@
|
|||
import React, { useMemo, useState } from "react"
|
||||
import React, { useContext, useMemo, useState } from "react"
|
||||
import Style from "./PageRepository.module.css"
|
||||
import classNames from "classnames"
|
||||
import BoxRepositoryTweets from "../components/interactive/BoxRepositoryTweets"
|
||||
import BoxWordcloud from "../components/interactive/BoxWordcloud"
|
||||
import BoxHeader from "../components/base/BoxHeader"
|
||||
import PickerVisualization from "../components/interactive/PickerVisualization"
|
||||
import PickerFilter from "../components/interactive/PickerFilter"
|
||||
|
@ -12,10 +11,17 @@ import { faFolder, faFolderOpen, faTrash } from "@fortawesome/free-solid-svg-ico
|
|||
import {FontAwesomeIcon} from "@fortawesome/react-fontawesome"
|
||||
import { useParams } from "react-router"
|
||||
import Loading from "../components/base/Loading"
|
||||
import BoxVisualizationStats from "../components/interactive/BoxVisualizationStats"
|
||||
import BoxVisualizationGraph from "../components/interactive/BoxVisualizationGraph"
|
||||
import BoxVisualizationMap from "../components/interactive/BoxVisualizationMap"
|
||||
import BoxVisualizationWordcloud from "../components/interactive/BoxVisualizationWordcloud"
|
||||
import BoxFull from "../components/base/BoxFull"
|
||||
import ContextLanguage from "../contexts/ContextLanguage"
|
||||
|
||||
|
||||
export default function PageRepository({ className, ...props }) {
|
||||
const {id} = useParams()
|
||||
const {strings} = useContext(ContextLanguage)
|
||||
|
||||
const [visualizationTab, setVisualizationTab] = useState("wordcloud")
|
||||
const [addFilterTab, setAddFilterTab] = useState("hashtag")
|
||||
|
@ -46,36 +52,6 @@ export default function PageRepository({ className, ...props }) {
|
|||
)
|
||||
const tweets = tweetsBv.resources && tweetsBv.error ? [] : tweetsBv.resources
|
||||
|
||||
const words = useMemo(
|
||||
() => {
|
||||
let preprocessedWords = {}
|
||||
for(const tweet of tweets) {
|
||||
if(!tweet.content) {
|
||||
continue
|
||||
}
|
||||
for(const word of tweet.content.toLowerCase().split(/\s+/)) {
|
||||
if(!preprocessedWords.hasOwnProperty(word)) {
|
||||
preprocessedWords[word] = 0
|
||||
}
|
||||
preprocessedWords[word] += 1
|
||||
}
|
||||
}
|
||||
|
||||
let processedWords = []
|
||||
for(const word in preprocessedWords) {
|
||||
if(!preprocessedWords.hasOwnProperty(word)) {
|
||||
continue
|
||||
}
|
||||
processedWords.push({
|
||||
text: word,
|
||||
value: preprocessedWords[word]
|
||||
})
|
||||
}
|
||||
return processedWords
|
||||
},
|
||||
[tweets]
|
||||
)
|
||||
|
||||
let contents;
|
||||
if(!repositoryBr.firstLoad || !tweetsBv.firstLoad) {
|
||||
contents = <>
|
||||
|
@ -85,8 +61,6 @@ export default function PageRepository({ className, ...props }) {
|
|||
</>
|
||||
}
|
||||
else if(repository === null) {
|
||||
console.debug("repositoryBr: ", repositoryBr, ", tweetsBv: ", tweetsBv)
|
||||
|
||||
// TODO: Translate this!
|
||||
contents = <>
|
||||
<BoxHeader className={Style.Header}>
|
||||
|
@ -111,17 +85,44 @@ export default function PageRepository({ className, ...props }) {
|
|||
setTab={setVisualizationTab}
|
||||
/>
|
||||
{visualizationTab === "wordcloud" ?
|
||||
<BoxWordcloud
|
||||
className={Style.Wordcloud}
|
||||
words={words}
|
||||
/>
|
||||
: null}
|
||||
<BoxVisualizationWordcloud
|
||||
className={Style.Wordcloud}
|
||||
tweets={tweets}
|
||||
/>
|
||||
: null}
|
||||
{visualizationTab === "histogram" ?
|
||||
<BoxVisualizationGraph
|
||||
className={Style.Wordcloud}
|
||||
tweets={tweets}
|
||||
/>
|
||||
: null}
|
||||
{visualizationTab === "map" ?
|
||||
<BoxVisualizationMap
|
||||
className={Style.Wordcloud}
|
||||
tweets={tweets}
|
||||
/>
|
||||
: null}
|
||||
{visualizationTab === "stats" ?
|
||||
<BoxVisualizationStats
|
||||
className={Style.Wordcloud}
|
||||
tweets={tweets}
|
||||
totalTweetCount={tweets.length}
|
||||
/>
|
||||
: null}
|
||||
|
||||
<PickerFilter
|
||||
className={Style.FilterPicker}
|
||||
currentTab={addFilterTab}
|
||||
setTab={setAddFilterTab}
|
||||
/>
|
||||
|
||||
<BoxFull header={strings.notImplemented} className={Style.Filters}>
|
||||
{strings.notImplemented}
|
||||
</BoxFull>
|
||||
|
||||
<BoxFull header={strings.notImplemented} className={Style.AddFilter}>
|
||||
{strings.notImplemented}
|
||||
</BoxFull>
|
||||
</>
|
||||
}
|
||||
|
||||
|
|
35
nest_frontend/utils/tokenizeTweetWords.js
Normal file
35
nest_frontend/utils/tokenizeTweetWords.js
Normal file
|
@ -0,0 +1,35 @@
|
|||
import sw from "stopword"
|
||||
|
||||
|
||||
const stopwords = [...sw.it, ...sw.en, "rt"]
|
||||
|
||||
|
||||
export default function(tweets = {}) {
|
||||
let preprocessedWords = {}
|
||||
for(const tweet of tweets) {
|
||||
if(!tweet.content) {
|
||||
continue
|
||||
}
|
||||
for(const word of tweet.content.toLowerCase().split(/\s+/)) {
|
||||
if(stopwords.includes(word)) continue
|
||||
if(word.startsWith("https://")) continue
|
||||
|
||||
if(!preprocessedWords.hasOwnProperty(word)) {
|
||||
preprocessedWords[word] = 0
|
||||
}
|
||||
preprocessedWords[word] += 1
|
||||
}
|
||||
}
|
||||
|
||||
let processedWords = []
|
||||
for(const word in preprocessedWords) {
|
||||
if(!preprocessedWords.hasOwnProperty(word)) {
|
||||
continue
|
||||
}
|
||||
processedWords.push({
|
||||
text: word,
|
||||
value: preprocessedWords[word]
|
||||
})
|
||||
}
|
||||
return processedWords
|
||||
}
|
11
package-lock.json
generated
11
package-lock.json
generated
|
@ -28,6 +28,7 @@
|
|||
"react-router-dom": "^5.2.0",
|
||||
"react-scripts": "4.0.3",
|
||||
"serve": "^11.3.2",
|
||||
"stopword": "^1.0.7",
|
||||
"web-vitals": "^1.1.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
@ -18917,6 +18918,11 @@
|
|||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/stopword": {
|
||||
"version": "1.0.7",
|
||||
"resolved": "https://registry.npmjs.org/stopword/-/stopword-1.0.7.tgz",
|
||||
"integrity": "sha512-KKPM5LGeulAxoNrg384AfNSoUvphG87sgj/vTHOkJ5M5U7hO99XA6Sl0HBhtCBv17Tv3kQPLj+5fCt7cadjXWQ=="
|
||||
},
|
||||
"node_modules/stream-browserify": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.2.tgz",
|
||||
|
@ -37540,6 +37546,11 @@
|
|||
"resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz",
|
||||
"integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks="
|
||||
},
|
||||
"stopword": {
|
||||
"version": "1.0.7",
|
||||
"resolved": "https://registry.npmjs.org/stopword/-/stopword-1.0.7.tgz",
|
||||
"integrity": "sha512-KKPM5LGeulAxoNrg384AfNSoUvphG87sgj/vTHOkJ5M5U7hO99XA6Sl0HBhtCBv17Tv3kQPLj+5fCt7cadjXWQ=="
|
||||
},
|
||||
"stream-browserify": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.2.tgz",
|
||||
|
|
|
@ -23,6 +23,7 @@
|
|||
"react-router-dom": "^5.2.0",
|
||||
"react-scripts": "4.0.3",
|
||||
"serve": "^11.3.2",
|
||||
"stopword": "^1.0.7",
|
||||
"web-vitals": "^1.1.1"
|
||||
},
|
||||
"main": "nest_frontend/index.js",
|
||||
|
|
Loading…
Reference in a new issue