✨ Implement analysis visualization

nest_frontend/LocalizationStrings.js

@@ -46,7 +46,7 @@ export default {
         repoEdit: "Modifica repository",
         menuActive: "Le tue repository attive",
         menuArchived: "Le tue repository archiviate",
-        emptyMenu: "Non c'è nulla qui",
+        emptyMenu: "Non c'è nulla qui.",
         delete: "Elimina",
         archive: "Archivia",
         edit: "Modifica",
@@ -117,7 +117,7 @@ export default {
         repoEdit: "Edit repository",
         menuActive: "Your active repositories",
         menuArchived: "Your archived repositories",
-        emptyMenu: "There's nothing here",
+        emptyMenu: "There's nothing here.",
         delete: "Delete",
         archive: "Archive",
         edit: "Edit",
@@ -188,7 +188,7 @@ export default {
         repoEdit: "Muokkaa arkistoa",
         menuActive: "Aktiiviset arkistosi",
         menuArchived: "Arkistoidut arkistosi",
-        emptyMenu: "Täällä ei ole mitään",
+        emptyMenu: "Täällä ei ole mitään.",
         delete: "Poista",
         archive: "Arkistoi",
         edit: "Muokkaa",

nest_frontend/components/base/BoxMap.js

@@ -4,7 +4,7 @@ import BoxFull from "./BoxFull"
 import { MapContainer, TileLayer } from "react-leaflet"
 
 
-export default function BoxMap({ header, setMap, startingPosition, startingZoom, button, children, additions, ...props }) {
+export default function BoxMap({ header, setMap, startingPosition = { lat: 41.89309, lng: 12.48289 }, startingZoom = 3, button, children, ...props }) {
     return (
         <BoxFull
             header={header}
@@ -21,7 +21,7 @@ export default function BoxMap({ header, setMap, startingPosition, startingZoom,
                     attribution='(c) <a href="https://osm.org/copyright">OpenStreetMap contributors</a>'
                     url="https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png"
                 />
-                {additions}
+                {children}
                 <div className={"leaflet-top leaflet-right"}>
                     <div className={"leaflet-control"}>
                         {button}

nest_frontend/components/base/BoxWordcloud.js

@@ -1,7 +1,7 @@
-import React, { useContext } from "react"
+import React from "react"
 import BoxFull from "../base/BoxFull"
 import ReactWordcloud from "@steffo/nest-react-wordcloud"
-import ContextLanguage from "../../contexts/ContextLanguage"
+import Style from "./BoxWordcloud.module.css"
 
 
 /**
@@ -12,19 +12,17 @@ import ContextLanguage from "../../contexts/ContextLanguage"
  * @returns {JSX.Element}
  * @constructor
  */
-export default function BoxWordcloud({ words, props }) {
+export default function BoxWordcloud({ words, ...props }) {
-    const { strings } = useContext(ContextLanguage)
-
     return (
-        <BoxFull header={strings.wordcloud} {...props}>
+        <BoxFull {...props}>
-            <div style={{ "width": "100%", "height": "100%" }}>
+            <div className={Style.WordcloudContainer}>
                 <ReactWordcloud
                     options={{
                         colors: [
                             "var(--fg-primary)",
                         ],
                         fontFamily: "Bree Serif",
-                        fontSizes: [16, 48],
+                        fontSizes: [8, 64],
                         size: undefined,
                         deterministic: true,
                     }}

nest_frontend/components/base/BoxWordcloud.module.css

@@ -0,0 +1,4 @@
+.WordcloudContainer {
+    width: 100%;
+    height: 100%;
+}

nest_frontend/components/base/summary/SummaryLeft.module.css

@@ -1,6 +1,6 @@
 .SummaryLeft {
-    min-width: 175px;
+    width: 260px;
-    width: 275px;
+    flex-shrink: 0;
 
     display: grid;
     grid-template-areas:

nest_frontend/components/interactive/BoxConditionMap.js

@@ -8,9 +8,6 @@ import ContextLanguage from "../../contexts/ContextLanguage"
 import BoxMap from "../base/BoxMap"
 
 
-const STARTING_POSITION = { lat: 41.89309, lng: 12.48289 }
-const STARTING_ZOOM = 3
-
 /**
  * https://wiki.openstreetmap.org/wiki/Zoom_levels
  */
@@ -47,8 +44,8 @@ const MPIXEL = [
  * @constructor
  */
 export default function BoxConditionMap({ ...props }) {
-    const [position, setPosition] = useState(STARTING_POSITION)
+    const [position, setPosition] = useState()
-    const [zoom, setZoom] = useState(STARTING_ZOOM)
+    const [zoom, setZoom] = useState()
     const [map, setMap] = useState(null)
     const { addCondition } = useRepositoryEditor()
     const { strings } = useContext(ContextLanguage)
@@ -92,7 +89,6 @@ export default function BoxConditionMap({ ...props }) {
             "COORDINATES",
             `< ${radius} ${position.lat} ${position.lng}`,
         ))
-        setPosition(STARTING_POSITION)
     }
 
     return (
@@ -106,8 +102,6 @@ export default function BoxConditionMap({ ...props }) {
                     {strings.byZone}
                 </span>
             }
-            startingPosition={STARTING_POSITION}
-            startingZoom={STARTING_ZOOM}
             setMap={setMap}
             button={
                 <ButtonIconOnly

nest_frontend/components/interactive/BoxRepositories.js

@@ -3,6 +3,7 @@ import BoxFullScrollable from "../base/BoxFullScrollable"
 import Loading from "../base/Loading"
 import ContextLanguage from "../../contexts/ContextLanguage"
 import SummaryRepository from "./SummaryRepository"
+import Empty from "./Empty"
 
 
 /**
@@ -21,14 +22,12 @@ import SummaryRepository from "./SummaryRepository"
  * @constructor
  */
 export default function BoxRepositories({ repositories, view, archive, edit, destroy, loading, running, className, ...props }) {
-    const { strings } = useContext(ContextLanguage)
-
     let contents
     if(loading) {
         contents = <Loading/>
     }
     else if(repositories.length === 0) {
-        contents = <i>{strings.emptyMenu}</i>
+        contents = <Empty/>
     }
     else {
         contents = repositories.map(repo => (

nest_frontend/components/interactive/BoxRepositoryTweets.js

@@ -1,14 +1,23 @@
 import React from "react"
 import BoxFullScrollable from "../base/BoxFullScrollable"
 import SummaryTweet from "./SummaryTweet"
+import Empty from "./Empty"
 
 
 export default function BoxRepositoryTweets({ tweets, ...props }) {
     // TODO: Translate this
 
+    let content
+    if(tweets.length === 0) {
+        content = <Empty/>
+    }
+    else {
+        content = tweets.map(tweet => <SummaryTweet key={tweet["snowflake"]} tweet={tweet}/>)
+    }
+
     return (
         <BoxFullScrollable header={"Tweets"} {...props}>
-            {tweets.map(tweet => <SummaryTweet key={tweet.snowflake} tweet={tweet}/>)}
+            {content}
         </BoxFullScrollable>
     )
 }

nest_frontend/components/interactive/BoxVisualizationGraph.js

@@ -0,0 +1,14 @@
+import React from "react"
+import BoxFull from "../base/BoxFull"
+
+
+export default function BoxVisualizationGraph({ children, className, ...props }) {
+    // TODO: translate this
+    // TODO: implement this
+
+    return (
+        <BoxFull header={"Hourly graph"} {...props}>
+            {children}
+        </BoxFull>
+    )
+}

nest_frontend/components/interactive/BoxVisualizationMap.js

@@ -0,0 +1,40 @@
+import React, { useContext } from "react"
+import BoxMap from "../base/BoxMap"
+import ContextLanguage from "../../contexts/ContextLanguage"
+import { Marker, Popup } from "react-leaflet"
+
+
+const locationRegex = /[{](?<lat>[0-9.]+),(?<lng>[0-9.]+)[}]/
+
+export default function BoxVisualizationMap({ tweets, ...props }) {
+    // TODO: translate this
+    const {strings} = useContext(ContextLanguage)
+
+    console.debug(tweets)
+    const markers = tweets.filter(tweet => tweet.location).map(tweet => {
+        const match = locationRegex.exec(tweet.location)
+        if(!match) {
+            console.error("No match for location ", tweet.location)
+            return null
+        }
+        const {lat, lng} = match.groups
+        return (
+            <Marker key={tweet["snowflake"]} position={[Number.parseFloat(lat), Number.parseFloat(lng)]}>
+                <Popup>
+                    <p>
+                        {tweet["content"]}
+                    </p>
+                    <p>
+                        — <a href={`https://twitter.com/${tweet["poster"]}/status/${tweet["snowflake"]}`}>@{tweet["poster"]}</a>
+                    </p>
+                </Popup>
+            </Marker>
+        )
+    })
+
+    return (
+        <BoxMap header={"Map"} {...props}>
+            {markers}
+        </BoxMap>
+    )
+}

nest_frontend/components/interactive/BoxVisualizationStats.js

@@ -0,0 +1,77 @@
+import React, { useMemo } from "react"
+import FormLabelled from "../base/FormLabelled"
+import FormLabel from "../base/formparts/FormLabel"
+import BoxFullScrollable from "../base/BoxFullScrollable"
+import tokenizeTweetWords from "../../utils/tokenizeTweetWords"
+
+
+export default function BoxVisualizationStats({ tweets, totalTweetCount, ...props }) {
+
+    const words = useMemo(
+        () => tokenizeTweetWords(tweets),
+        [tweets]
+    )
+
+    const tweetCount = tweets.length
+    const tweetPct = tweetCount / totalTweetCount * 100
+    const tweetLocationCount = tweets.filter(tweet => tweet.location).length
+    const tweetLocationPct = tweetLocationCount / tweetCount * 100
+    const tweetContent = tweets.filter(tweet => tweet.content)
+    const tweetContentCount = tweetContent.length
+    const tweetContentPct = tweetContentCount / tweetCount * 100
+    const wordCount = words.map(word => word.value).reduce((a, b) => a+b)
+    const mostPopularWord = words.sort((wa, wb) => {
+        if(wa.value > wb.value) return -1
+        if(wa.value < wb.value) return 1
+        return 0
+    })[0].text
+    const users = [...new Set(tweets.map(tweet => tweet.poster))]
+    const usersCount = users.length
+
+    // TODO: tweets with picture count
+    // TODO: tweets with picture pct
+
+    // TODO: translate this
+    return (
+        <BoxFullScrollable header={"Stats"} {...props}>
+            <FormLabelled>
+                <FormLabel text={"Total tweets"}>
+                    <b>{totalTweetCount}</b>
+                </FormLabel>
+                <FormLabel text={"Displayed tweets"}>
+                    <b>{tweetCount}</b>
+                </FormLabel>
+                <FormLabel text={"% of displayed tweets"}>
+                    <b>{tweetPct.toFixed(2)}%</b>
+                </FormLabel>
+                <FormLabel text={"Tweets with location"}>
+                    <b>{tweetLocationCount}</b>
+                </FormLabel>
+                <FormLabel text={"% of tweets with location"}>
+                    <b>{tweetLocationPct.toFixed(2)}%</b>
+                </FormLabel>
+                <FormLabel text={"Tweets with content"}>
+                    <b>{tweetContentCount}</b>
+                </FormLabel>
+                <FormLabel text={"% of tweets with content"}>
+                    <b>{tweetContentPct.toFixed(2)}%</b>
+                </FormLabel>
+                <FormLabel text={"Word count"}>
+                    <b>{wordCount}</b>
+                </FormLabel>
+                <FormLabel text={"Most popular word"}>
+                    <b>{mostPopularWord}</b>
+                </FormLabel>
+                <FormLabel text={"Tweets with image"}>
+                    <b>🚧</b>
+                </FormLabel>
+                <FormLabel text={"% of tweets with image"}>
+                    <b>🚧</b>
+                </FormLabel>
+                <FormLabel text={"Users count"}>
+                    <b>{usersCount}</b>
+                </FormLabel>
+            </FormLabelled>
+        </BoxFullScrollable>
+    )
+}

nest_frontend/components/interactive/BoxVisualizationWordcloud.js

@@ -0,0 +1,18 @@
+import React, { useContext, useMemo } from "react"
+import BoxWordcloud from "../base/BoxWordcloud"
+import ContextLanguage from "../../contexts/ContextLanguage"
+import tokenizeTweetWords from "../../utils/tokenizeTweetWords"
+
+
+export default function BoxVisualizationWordcloud({ tweets = [], ...props }) {
+    const {strings} = useContext(ContextLanguage)
+
+    const words = useMemo(
+        () => tokenizeTweetWords(tweets),
+        [tweets]
+    )
+
+    return (
+        <BoxWordcloud header={strings.wordcloud} words={words} {...props}/>
+    )
+}

nest_frontend/components/interactive/Empty.js

@@ -0,0 +1,15 @@
+import React, { useContext } from "react"
+import Style from "./Empty.module.css"
+import classNames from "classnames"
+import ContextLanguage from "../../contexts/ContextLanguage"
+
+
+export default function Empty({ children, className, ...props }) {
+    const { strings } = useContext(ContextLanguage)
+
+    return (
+        <i className={classNames(Style.Empty, className)} {...props}>
+            {strings.emptyMenu}
+        </i>
+    )
+}

nest_frontend/components/interactive/Empty.module.css

@@ -0,0 +1,3 @@
+.Empty {
+    opacity: 0.5;
+}

nest_frontend/components/interactive/SummaryTweet.js

@@ -1,15 +1,18 @@
 import React from "react"
 import SummaryBase from "../base/summary/SummaryBase"
 import SummaryLeft from "../base/summary/SummaryLeft"
-import { faComment, faMapPin } from "@fortawesome/free-solid-svg-icons"
+import { faComment, faLocationArrow, faMapMarker, faMapMarkerAlt, faMapPin } from "@fortawesome/free-solid-svg-icons"
 import SummaryText from "../base/summary/SummaryText"
 import SummaryRight from "../base/summary/SummaryRight"
 
 
 export default function SummaryTweet({ tweet, ...props }) {
     let icon
-    if(tweet.place) {
+    if(tweet["location"]) {
-        icon = faMapPin
+        icon = faMapMarkerAlt
+    }
+    else if(tweet["place"]) {
+        icon = faLocationArrow
     }
     else {
         icon = faComment
@@ -19,9 +22,9 @@ export default function SummaryTweet({ tweet, ...props }) {
         <SummaryBase {...props}>
             <SummaryLeft
                 icon={icon}
-                title={`@${tweet.poster}`}
+                title={`@${tweet["poster"]}`}
-                subtitle={tweet.place}
+                subtitle={new Date(tweet["insert_time"]).toLocaleString()}
-                onClick={() => window.open(`https://twitter.com/${tweet.poster}/status/${tweet.snowflake}`)}
+                onClick={() => window.open(`https://twitter.com/${tweet["poster"]}/status/${tweet["snowflake"]}`)}
             />
             <SummaryText>
                 {tweet.content}

nest_frontend/components/providers/RepositoryViewer.js

@@ -1,22 +0,0 @@
-import React  from "react"
-import Style from "./RepositoryViewer.module.css"
-import classNames from "classnames"
-import ContextRepositoryViewer from "../../contexts/ContextRepositoryViewer"
-
-
-export default function RepositoryViewer({
-                                             id,
-                                         }) {
-    return (
-        <ContextRepositoryViewer.Provider
-            value={{
-                id,
-
-            }}
-        >
-            <div className={classNames(Style.RepositoryViewer, className)}>
-
-            </div>
-        </ContextRepositoryViewer.Provider>
-    )
-}

nest_frontend/hooks/useRepositoryViewer.js

@@ -1,14 +0,0 @@
-import { useContext } from "react"
-import ContextRepositoryViewer from "../contexts/ContextRepositoryViewer"
-
-
-/**
- * Hook to quickly use {@link ContextRepositoryViewer}.
- */
-export default function useRepositoryViewer() {
-    const context = useContext(ContextRepositoryViewer)
-    if(!context) {
-        throw new Error("This component must be placed inside a RepositoryViewer.")
-    }
-    return context
-}

nest_frontend/routes/PageRepository.js

@@ -1,8 +1,7 @@
-import React, { useMemo, useState } from "react"
+import React, { useContext, useMemo, useState } from "react"
 import Style from "./PageRepository.module.css"
 import classNames from "classnames"
 import BoxRepositoryTweets from "../components/interactive/BoxRepositoryTweets"
-import BoxWordcloud from "../components/interactive/BoxWordcloud"
 import BoxHeader from "../components/base/BoxHeader"
 import PickerVisualization from "../components/interactive/PickerVisualization"
 import PickerFilter from "../components/interactive/PickerFilter"
@@ -12,10 +11,17 @@ import { faFolder, faFolderOpen, faTrash } from "@fortawesome/free-solid-svg-ico
 import {FontAwesomeIcon} from "@fortawesome/react-fontawesome"
 import { useParams } from "react-router"
 import Loading from "../components/base/Loading"
+import BoxVisualizationStats from "../components/interactive/BoxVisualizationStats"
+import BoxVisualizationGraph from "../components/interactive/BoxVisualizationGraph"
+import BoxVisualizationMap from "../components/interactive/BoxVisualizationMap"
+import BoxVisualizationWordcloud from "../components/interactive/BoxVisualizationWordcloud"
+import BoxFull from "../components/base/BoxFull"
+import ContextLanguage from "../contexts/ContextLanguage"
 
 
 export default function PageRepository({ className, ...props }) {
     const {id} = useParams()
+    const {strings} = useContext(ContextLanguage)
 
     const [visualizationTab, setVisualizationTab] = useState("wordcloud")
     const [addFilterTab, setAddFilterTab] = useState("hashtag")
@@ -46,36 +52,6 @@ export default function PageRepository({ className, ...props }) {
     )
     const tweets = tweetsBv.resources && tweetsBv.error ? [] : tweetsBv.resources
 
-    const words = useMemo(
-        () => {
-            let preprocessedWords = {}
-            for(const tweet of tweets) {
-                if(!tweet.content) {
-                    continue
-                }
-                for(const word of tweet.content.toLowerCase().split(/\s+/)) {
-                    if(!preprocessedWords.hasOwnProperty(word)) {
-                        preprocessedWords[word] = 0
-                    }
-                    preprocessedWords[word] += 1
-                }
-            }
-
-            let processedWords = []
-            for(const word in preprocessedWords) {
-                if(!preprocessedWords.hasOwnProperty(word)) {
-                    continue
-                }
-                processedWords.push({
-                    text: word,
-                    value: preprocessedWords[word]
-                })
-            }
-            return processedWords
-        },
-        [tweets]
-    )
-
     let contents;
     if(!repositoryBr.firstLoad || !tweetsBv.firstLoad) {
         contents = <>
@@ -85,8 +61,6 @@ export default function PageRepository({ className, ...props }) {
         </>
     }
     else if(repository === null) {
-        console.debug("repositoryBr: ", repositoryBr, ", tweetsBv: ", tweetsBv)
-
         // TODO: Translate this!
         contents = <>
             <BoxHeader className={Style.Header}>
@@ -111,9 +85,28 @@ export default function PageRepository({ className, ...props }) {
                 setTab={setVisualizationTab}
             />
             {visualizationTab === "wordcloud" ?
-             <BoxWordcloud
+                <BoxVisualizationWordcloud
                     className={Style.Wordcloud}
-                 words={words}
+                    tweets={tweets}
+                />
+            : null}
+            {visualizationTab === "histogram" ?
+                <BoxVisualizationGraph
+                    className={Style.Wordcloud}
+                    tweets={tweets}
+                />
+            : null}
+            {visualizationTab === "map" ?
+                <BoxVisualizationMap
+                    className={Style.Wordcloud}
+                    tweets={tweets}
+                />
+            : null}
+            {visualizationTab === "stats" ?
+                <BoxVisualizationStats
+                    className={Style.Wordcloud}
+                    tweets={tweets}
+                    totalTweetCount={tweets.length}
                 />
             : null}
 
@@ -122,6 +115,14 @@ export default function PageRepository({ className, ...props }) {
                 currentTab={addFilterTab}
                 setTab={setAddFilterTab}
             />
+
+            <BoxFull header={strings.notImplemented} className={Style.Filters}>
+                {strings.notImplemented}
+            </BoxFull>
+
+            <BoxFull header={strings.notImplemented} className={Style.AddFilter}>
+                {strings.notImplemented}
+            </BoxFull>
         </>
     }
 

nest_frontend/utils/tokenizeTweetWords.js

@@ -0,0 +1,35 @@
+import sw from "stopword"
+
+
+const stopwords = [...sw.it, ...sw.en, "rt"]
+
+
+export default function(tweets = {}) {
+    let preprocessedWords = {}
+    for(const tweet of tweets) {
+        if(!tweet.content) {
+            continue
+        }
+        for(const word of tweet.content.toLowerCase().split(/\s+/)) {
+            if(stopwords.includes(word)) continue
+            if(word.startsWith("https://")) continue
+
+            if(!preprocessedWords.hasOwnProperty(word)) {
+                preprocessedWords[word] = 0
+            }
+            preprocessedWords[word] += 1
+        }
+    }
+
+    let processedWords = []
+    for(const word in preprocessedWords) {
+        if(!preprocessedWords.hasOwnProperty(word)) {
+            continue
+        }
+        processedWords.push({
+            text: word,
+            value: preprocessedWords[word]
+        })
+    }
+    return processedWords
+}

package-lock.json

@@ -28,6 +28,7 @@
                 "react-router-dom": "^5.2.0",
                 "react-scripts": "4.0.3",
                 "serve": "^11.3.2",
+                "stopword": "^1.0.7",
                 "web-vitals": "^1.1.1"
             },
             "devDependencies": {
@@ -18917,6 +18918,11 @@
                 "node": ">=0.10.0"
             }
         },
+        "node_modules/stopword": {
+            "version": "1.0.7",
+            "resolved": "https://registry.npmjs.org/stopword/-/stopword-1.0.7.tgz",
+            "integrity": "sha512-KKPM5LGeulAxoNrg384AfNSoUvphG87sgj/vTHOkJ5M5U7hO99XA6Sl0HBhtCBv17Tv3kQPLj+5fCt7cadjXWQ=="
+        },
         "node_modules/stream-browserify": {
             "version": "2.0.2",
             "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.2.tgz",
@@ -37540,6 +37546,11 @@
             "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz",
             "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks="
         },
+        "stopword": {
+            "version": "1.0.7",
+            "resolved": "https://registry.npmjs.org/stopword/-/stopword-1.0.7.tgz",
+            "integrity": "sha512-KKPM5LGeulAxoNrg384AfNSoUvphG87sgj/vTHOkJ5M5U7hO99XA6Sl0HBhtCBv17Tv3kQPLj+5fCt7cadjXWQ=="
+        },
         "stream-browserify": {
             "version": "2.0.2",
             "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.2.tgz",

package.json

@@ -23,6 +23,7 @@
         "react-router-dom": "^5.2.0",
         "react-scripts": "4.0.3",
         "serve": "^11.3.2",
+        "stopword": "^1.0.7",
         "web-vitals": "^1.1.1"
     },
     "main": "nest_frontend/index.js",