1
Fork 0
mirror of https://github.com/pds-nest/nest.git synced 2024-11-25 06:24:19 +00:00

Implement analysis visualization

This commit is contained in:
Steffo 2021-05-20 11:39:40 +02:00
parent b58e49b5b3
commit b471fc17c0
Signed by: steffo
GPG key ID: 6965406171929D01
21 changed files with 294 additions and 108 deletions

View file

@ -46,7 +46,7 @@ export default {
repoEdit: "Modifica repository",
menuActive: "Le tue repository attive",
menuArchived: "Le tue repository archiviate",
emptyMenu: "Non c'è nulla qui",
emptyMenu: "Non c'è nulla qui.",
delete: "Elimina",
archive: "Archivia",
edit: "Modifica",
@ -117,7 +117,7 @@ export default {
repoEdit: "Edit repository",
menuActive: "Your active repositories",
menuArchived: "Your archived repositories",
emptyMenu: "There's nothing here",
emptyMenu: "There's nothing here.",
delete: "Delete",
archive: "Archive",
edit: "Edit",
@ -188,7 +188,7 @@ export default {
repoEdit: "Muokkaa arkistoa",
menuActive: "Aktiiviset arkistosi",
menuArchived: "Arkistoidut arkistosi",
emptyMenu: "Täällä ei ole mitään",
emptyMenu: "Täällä ei ole mitään.",
delete: "Poista",
archive: "Arkistoi",
edit: "Muokkaa",

View file

@ -4,7 +4,7 @@ import BoxFull from "./BoxFull"
import { MapContainer, TileLayer } from "react-leaflet"
export default function BoxMap({ header, setMap, startingPosition, startingZoom, button, children, additions, ...props }) {
export default function BoxMap({ header, setMap, startingPosition = { lat: 41.89309, lng: 12.48289 }, startingZoom = 3, button, children, ...props }) {
return (
<BoxFull
header={header}
@ -21,7 +21,7 @@ export default function BoxMap({ header, setMap, startingPosition, startingZoom,
attribution='(c) <a href="https://osm.org/copyright">OpenStreetMap contributors</a>'
url="https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png"
/>
{additions}
{children}
<div className={"leaflet-top leaflet-right"}>
<div className={"leaflet-control"}>
{button}

View file

@ -1,7 +1,7 @@
import React, { useContext } from "react"
import React from "react"
import BoxFull from "../base/BoxFull"
import ReactWordcloud from "@steffo/nest-react-wordcloud"
import ContextLanguage from "../../contexts/ContextLanguage"
import Style from "./BoxWordcloud.module.css"
/**
@ -12,19 +12,17 @@ import ContextLanguage from "../../contexts/ContextLanguage"
* @returns {JSX.Element}
* @constructor
*/
export default function BoxWordcloud({ words, props }) {
const { strings } = useContext(ContextLanguage)
export default function BoxWordcloud({ words, ...props }) {
return (
<BoxFull header={strings.wordcloud} {...props}>
<div style={{ "width": "100%", "height": "100%" }}>
<BoxFull {...props}>
<div className={Style.WordcloudContainer}>
<ReactWordcloud
options={{
colors: [
"var(--fg-primary)",
],
fontFamily: "Bree Serif",
fontSizes: [16, 48],
fontSizes: [8, 64],
size: undefined,
deterministic: true,
}}

View file

@ -0,0 +1,4 @@
.WordcloudContainer {
width: 100%;
height: 100%;
}

View file

@ -1,6 +1,6 @@
.SummaryLeft {
min-width: 175px;
width: 275px;
width: 260px;
flex-shrink: 0;
display: grid;
grid-template-areas:

View file

@ -8,9 +8,6 @@ import ContextLanguage from "../../contexts/ContextLanguage"
import BoxMap from "../base/BoxMap"
const STARTING_POSITION = { lat: 41.89309, lng: 12.48289 }
const STARTING_ZOOM = 3
/**
* https://wiki.openstreetmap.org/wiki/Zoom_levels
*/
@ -47,8 +44,8 @@ const MPIXEL = [
* @constructor
*/
export default function BoxConditionMap({ ...props }) {
const [position, setPosition] = useState(STARTING_POSITION)
const [zoom, setZoom] = useState(STARTING_ZOOM)
const [position, setPosition] = useState()
const [zoom, setZoom] = useState()
const [map, setMap] = useState(null)
const { addCondition } = useRepositoryEditor()
const { strings } = useContext(ContextLanguage)
@ -92,7 +89,6 @@ export default function BoxConditionMap({ ...props }) {
"COORDINATES",
`< ${radius} ${position.lat} ${position.lng}`,
))
setPosition(STARTING_POSITION)
}
return (
@ -106,8 +102,6 @@ export default function BoxConditionMap({ ...props }) {
{strings.byZone}
</span>
}
startingPosition={STARTING_POSITION}
startingZoom={STARTING_ZOOM}
setMap={setMap}
button={
<ButtonIconOnly

View file

@ -3,6 +3,7 @@ import BoxFullScrollable from "../base/BoxFullScrollable"
import Loading from "../base/Loading"
import ContextLanguage from "../../contexts/ContextLanguage"
import SummaryRepository from "./SummaryRepository"
import Empty from "./Empty"
/**
@ -21,14 +22,12 @@ import SummaryRepository from "./SummaryRepository"
* @constructor
*/
export default function BoxRepositories({ repositories, view, archive, edit, destroy, loading, running, className, ...props }) {
const { strings } = useContext(ContextLanguage)
let contents
if(loading) {
contents = <Loading/>
}
else if(repositories.length === 0) {
contents = <i>{strings.emptyMenu}</i>
contents = <Empty/>
}
else {
contents = repositories.map(repo => (

View file

@ -1,14 +1,23 @@
import React from "react"
import BoxFullScrollable from "../base/BoxFullScrollable"
import SummaryTweet from "./SummaryTweet"
import Empty from "./Empty"
export default function BoxRepositoryTweets({ tweets, ...props }) {
// TODO: Translate this
let content
if(tweets.length === 0) {
content = <Empty/>
}
else {
content = tweets.map(tweet => <SummaryTweet key={tweet["snowflake"]} tweet={tweet}/>)
}
return (
<BoxFullScrollable header={"Tweets"} {...props}>
{tweets.map(tweet => <SummaryTweet key={tweet.snowflake} tweet={tweet}/>)}
{content}
</BoxFullScrollable>
)
}

View file

@ -0,0 +1,14 @@
import React from "react"
import BoxFull from "../base/BoxFull"
export default function BoxVisualizationGraph({ children, className, ...props }) {
// TODO: translate this
// TODO: implement this
return (
<BoxFull header={"Hourly graph"} {...props}>
{children}
</BoxFull>
)
}

View file

@ -0,0 +1,40 @@
import React, { useContext } from "react"
import BoxMap from "../base/BoxMap"
import ContextLanguage from "../../contexts/ContextLanguage"
import { Marker, Popup } from "react-leaflet"
const locationRegex = /[{](?<lat>[0-9.]+),(?<lng>[0-9.]+)[}]/
export default function BoxVisualizationMap({ tweets, ...props }) {
// TODO: translate this
const {strings} = useContext(ContextLanguage)
console.debug(tweets)
const markers = tweets.filter(tweet => tweet.location).map(tweet => {
const match = locationRegex.exec(tweet.location)
if(!match) {
console.error("No match for location ", tweet.location)
return null
}
const {lat, lng} = match.groups
return (
<Marker key={tweet["snowflake"]} position={[Number.parseFloat(lat), Number.parseFloat(lng)]}>
<Popup>
<p>
{tweet["content"]}
</p>
<p>
<a href={`https://twitter.com/${tweet["poster"]}/status/${tweet["snowflake"]}`}>@{tweet["poster"]}</a>
</p>
</Popup>
</Marker>
)
})
return (
<BoxMap header={"Map"} {...props}>
{markers}
</BoxMap>
)
}

View file

@ -0,0 +1,77 @@
import React, { useMemo } from "react"
import FormLabelled from "../base/FormLabelled"
import FormLabel from "../base/formparts/FormLabel"
import BoxFullScrollable from "../base/BoxFullScrollable"
import tokenizeTweetWords from "../../utils/tokenizeTweetWords"
export default function BoxVisualizationStats({ tweets, totalTweetCount, ...props }) {
const words = useMemo(
() => tokenizeTweetWords(tweets),
[tweets]
)
const tweetCount = tweets.length
const tweetPct = tweetCount / totalTweetCount * 100
const tweetLocationCount = tweets.filter(tweet => tweet.location).length
const tweetLocationPct = tweetLocationCount / tweetCount * 100
const tweetContent = tweets.filter(tweet => tweet.content)
const tweetContentCount = tweetContent.length
const tweetContentPct = tweetContentCount / tweetCount * 100
const wordCount = words.map(word => word.value).reduce((a, b) => a+b)
const mostPopularWord = words.sort((wa, wb) => {
if(wa.value > wb.value) return -1
if(wa.value < wb.value) return 1
return 0
})[0].text
const users = [...new Set(tweets.map(tweet => tweet.poster))]
const usersCount = users.length
// TODO: tweets with picture count
// TODO: tweets with picture pct
// TODO: translate this
return (
<BoxFullScrollable header={"Stats"} {...props}>
<FormLabelled>
<FormLabel text={"Total tweets"}>
<b>{totalTweetCount}</b>
</FormLabel>
<FormLabel text={"Displayed tweets"}>
<b>{tweetCount}</b>
</FormLabel>
<FormLabel text={"% of displayed tweets"}>
<b>{tweetPct.toFixed(2)}%</b>
</FormLabel>
<FormLabel text={"Tweets with location"}>
<b>{tweetLocationCount}</b>
</FormLabel>
<FormLabel text={"% of tweets with location"}>
<b>{tweetLocationPct.toFixed(2)}%</b>
</FormLabel>
<FormLabel text={"Tweets with content"}>
<b>{tweetContentCount}</b>
</FormLabel>
<FormLabel text={"% of tweets with content"}>
<b>{tweetContentPct.toFixed(2)}%</b>
</FormLabel>
<FormLabel text={"Word count"}>
<b>{wordCount}</b>
</FormLabel>
<FormLabel text={"Most popular word"}>
<b>{mostPopularWord}</b>
</FormLabel>
<FormLabel text={"Tweets with image"}>
<b>🚧</b>
</FormLabel>
<FormLabel text={"% of tweets with image"}>
<b>🚧</b>
</FormLabel>
<FormLabel text={"Users count"}>
<b>{usersCount}</b>
</FormLabel>
</FormLabelled>
</BoxFullScrollable>
)
}

View file

@ -0,0 +1,18 @@
import React, { useContext, useMemo } from "react"
import BoxWordcloud from "../base/BoxWordcloud"
import ContextLanguage from "../../contexts/ContextLanguage"
import tokenizeTweetWords from "../../utils/tokenizeTweetWords"
export default function BoxVisualizationWordcloud({ tweets = [], ...props }) {
const {strings} = useContext(ContextLanguage)
const words = useMemo(
() => tokenizeTweetWords(tweets),
[tweets]
)
return (
<BoxWordcloud header={strings.wordcloud} words={words} {...props}/>
)
}

View file

@ -0,0 +1,15 @@
import React, { useContext } from "react"
import Style from "./Empty.module.css"
import classNames from "classnames"
import ContextLanguage from "../../contexts/ContextLanguage"
export default function Empty({ children, className, ...props }) {
const { strings } = useContext(ContextLanguage)
return (
<i className={classNames(Style.Empty, className)} {...props}>
{strings.emptyMenu}
</i>
)
}

View file

@ -0,0 +1,3 @@
.Empty {
opacity: 0.5;
}

View file

@ -1,15 +1,18 @@
import React from "react"
import SummaryBase from "../base/summary/SummaryBase"
import SummaryLeft from "../base/summary/SummaryLeft"
import { faComment, faMapPin } from "@fortawesome/free-solid-svg-icons"
import { faComment, faLocationArrow, faMapMarker, faMapMarkerAlt, faMapPin } from "@fortawesome/free-solid-svg-icons"
import SummaryText from "../base/summary/SummaryText"
import SummaryRight from "../base/summary/SummaryRight"
export default function SummaryTweet({ tweet, ...props }) {
let icon
if(tweet.place) {
icon = faMapPin
if(tweet["location"]) {
icon = faMapMarkerAlt
}
else if(tweet["place"]) {
icon = faLocationArrow
}
else {
icon = faComment
@ -19,9 +22,9 @@ export default function SummaryTweet({ tweet, ...props }) {
<SummaryBase {...props}>
<SummaryLeft
icon={icon}
title={`@${tweet.poster}`}
subtitle={tweet.place}
onClick={() => window.open(`https://twitter.com/${tweet.poster}/status/${tweet.snowflake}`)}
title={`@${tweet["poster"]}`}
subtitle={new Date(tweet["insert_time"]).toLocaleString()}
onClick={() => window.open(`https://twitter.com/${tweet["poster"]}/status/${tweet["snowflake"]}`)}
/>
<SummaryText>
{tweet.content}

View file

@ -1,22 +0,0 @@
import React from "react"
import Style from "./RepositoryViewer.module.css"
import classNames from "classnames"
import ContextRepositoryViewer from "../../contexts/ContextRepositoryViewer"
export default function RepositoryViewer({
id,
}) {
return (
<ContextRepositoryViewer.Provider
value={{
id,
}}
>
<div className={classNames(Style.RepositoryViewer, className)}>
</div>
</ContextRepositoryViewer.Provider>
)
}

View file

@ -1,14 +0,0 @@
import { useContext } from "react"
import ContextRepositoryViewer from "../contexts/ContextRepositoryViewer"
/**
* Hook to quickly use {@link ContextRepositoryViewer}.
*/
export default function useRepositoryViewer() {
const context = useContext(ContextRepositoryViewer)
if(!context) {
throw new Error("This component must be placed inside a RepositoryViewer.")
}
return context
}

View file

@ -1,8 +1,7 @@
import React, { useMemo, useState } from "react"
import React, { useContext, useMemo, useState } from "react"
import Style from "./PageRepository.module.css"
import classNames from "classnames"
import BoxRepositoryTweets from "../components/interactive/BoxRepositoryTweets"
import BoxWordcloud from "../components/interactive/BoxWordcloud"
import BoxHeader from "../components/base/BoxHeader"
import PickerVisualization from "../components/interactive/PickerVisualization"
import PickerFilter from "../components/interactive/PickerFilter"
@ -12,10 +11,17 @@ import { faFolder, faFolderOpen, faTrash } from "@fortawesome/free-solid-svg-ico
import {FontAwesomeIcon} from "@fortawesome/react-fontawesome"
import { useParams } from "react-router"
import Loading from "../components/base/Loading"
import BoxVisualizationStats from "../components/interactive/BoxVisualizationStats"
import BoxVisualizationGraph from "../components/interactive/BoxVisualizationGraph"
import BoxVisualizationMap from "../components/interactive/BoxVisualizationMap"
import BoxVisualizationWordcloud from "../components/interactive/BoxVisualizationWordcloud"
import BoxFull from "../components/base/BoxFull"
import ContextLanguage from "../contexts/ContextLanguage"
export default function PageRepository({ className, ...props }) {
const {id} = useParams()
const {strings} = useContext(ContextLanguage)
const [visualizationTab, setVisualizationTab] = useState("wordcloud")
const [addFilterTab, setAddFilterTab] = useState("hashtag")
@ -46,36 +52,6 @@ export default function PageRepository({ className, ...props }) {
)
const tweets = tweetsBv.resources && tweetsBv.error ? [] : tweetsBv.resources
const words = useMemo(
() => {
let preprocessedWords = {}
for(const tweet of tweets) {
if(!tweet.content) {
continue
}
for(const word of tweet.content.toLowerCase().split(/\s+/)) {
if(!preprocessedWords.hasOwnProperty(word)) {
preprocessedWords[word] = 0
}
preprocessedWords[word] += 1
}
}
let processedWords = []
for(const word in preprocessedWords) {
if(!preprocessedWords.hasOwnProperty(word)) {
continue
}
processedWords.push({
text: word,
value: preprocessedWords[word]
})
}
return processedWords
},
[tweets]
)
let contents;
if(!repositoryBr.firstLoad || !tweetsBv.firstLoad) {
contents = <>
@ -85,8 +61,6 @@ export default function PageRepository({ className, ...props }) {
</>
}
else if(repository === null) {
console.debug("repositoryBr: ", repositoryBr, ", tweetsBv: ", tweetsBv)
// TODO: Translate this!
contents = <>
<BoxHeader className={Style.Header}>
@ -111,17 +85,44 @@ export default function PageRepository({ className, ...props }) {
setTab={setVisualizationTab}
/>
{visualizationTab === "wordcloud" ?
<BoxWordcloud
className={Style.Wordcloud}
words={words}
/>
: null}
<BoxVisualizationWordcloud
className={Style.Wordcloud}
tweets={tweets}
/>
: null}
{visualizationTab === "histogram" ?
<BoxVisualizationGraph
className={Style.Wordcloud}
tweets={tweets}
/>
: null}
{visualizationTab === "map" ?
<BoxVisualizationMap
className={Style.Wordcloud}
tweets={tweets}
/>
: null}
{visualizationTab === "stats" ?
<BoxVisualizationStats
className={Style.Wordcloud}
tweets={tweets}
totalTweetCount={tweets.length}
/>
: null}
<PickerFilter
className={Style.FilterPicker}
currentTab={addFilterTab}
setTab={setAddFilterTab}
/>
<BoxFull header={strings.notImplemented} className={Style.Filters}>
{strings.notImplemented}
</BoxFull>
<BoxFull header={strings.notImplemented} className={Style.AddFilter}>
{strings.notImplemented}
</BoxFull>
</>
}

View file

@ -0,0 +1,35 @@
import sw from "stopword"
const stopwords = [...sw.it, ...sw.en, "rt"]
export default function(tweets = {}) {
let preprocessedWords = {}
for(const tweet of tweets) {
if(!tweet.content) {
continue
}
for(const word of tweet.content.toLowerCase().split(/\s+/)) {
if(stopwords.includes(word)) continue
if(word.startsWith("https://")) continue
if(!preprocessedWords.hasOwnProperty(word)) {
preprocessedWords[word] = 0
}
preprocessedWords[word] += 1
}
}
let processedWords = []
for(const word in preprocessedWords) {
if(!preprocessedWords.hasOwnProperty(word)) {
continue
}
processedWords.push({
text: word,
value: preprocessedWords[word]
})
}
return processedWords
}

11
package-lock.json generated
View file

@ -28,6 +28,7 @@
"react-router-dom": "^5.2.0",
"react-scripts": "4.0.3",
"serve": "^11.3.2",
"stopword": "^1.0.7",
"web-vitals": "^1.1.1"
},
"devDependencies": {
@ -18917,6 +18918,11 @@
"node": ">=0.10.0"
}
},
"node_modules/stopword": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/stopword/-/stopword-1.0.7.tgz",
"integrity": "sha512-KKPM5LGeulAxoNrg384AfNSoUvphG87sgj/vTHOkJ5M5U7hO99XA6Sl0HBhtCBv17Tv3kQPLj+5fCt7cadjXWQ=="
},
"node_modules/stream-browserify": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.2.tgz",
@ -37540,6 +37546,11 @@
"resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz",
"integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks="
},
"stopword": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/stopword/-/stopword-1.0.7.tgz",
"integrity": "sha512-KKPM5LGeulAxoNrg384AfNSoUvphG87sgj/vTHOkJ5M5U7hO99XA6Sl0HBhtCBv17Tv3kQPLj+5fCt7cadjXWQ=="
},
"stream-browserify": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.2.tgz",

View file

@ -23,6 +23,7 @@
"react-router-dom": "^5.2.0",
"react-scripts": "4.0.3",
"serve": "^11.3.2",
"stopword": "^1.0.7",
"web-vitals": "^1.1.1"
},
"main": "nest_frontend/index.js",