Maybe like this?

.github/workflows/publish.yml

@@ -10,15 +10,11 @@ jobs:
     steps:
     - name: Checkout the repository
       uses: actions/checkout@v2
-    - name: Setup Python and Poetry
+    - name: Build and publish the package
       uses: abatilo/actions-poetry@v1.5.0
+      env:
+        POETRY_PYPI_TOKEN_PYPI: ${{ secrets.POETRY_PYPI_TOKEN_PYPI }}
       with:
         python_version: 3.8
         poetry_version: 1.0
-        args: install
-    - name: Build
-      run: poetry build
-    - name: Publish
-      env:
-        POETRY_PYPI_TOKEN_PYPI: ${{ secrets.POETRY_PYPI_TOKEN_PYPI }}
-      run: poetry publish
+        args: publish --build