Skip to content

Commit

Permalink
fix Commento top-level comments import
Browse files Browse the repository at this point in the history 
Previously, top-level comments were incorrectly assigned
parent comment id "root", which made them non-root,
so they are not returned when requested
in the `/find?format=tree` API call.

To fix the previously imported comments, please export all your comments
and replace `"pid":"root"` with `"pid":""` and then re-import them.
  • Loading branch information
@paskal
paskal committed Nov 18, 2023
1 parent cd481d4 commit ce678bf
Show file tree
Hide file tree
Showing 3 changed files with 275 additions and 1 deletion.
8 changes: 7 additions & 1 deletion backend/app/migrator/commento.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,12 @@ func (d *Commento) convert(r io.Reader, siteID string) (ch chan store.Comment) {
continue
}

parentID := comment.ParentHex
// comments with ParentHex == "root" are top-level comments
if parentID == "root" {
parentID = ""
}

c := store.Comment{
ID: comment.CommentHex,
Locator: store.Locator{
Expand All @@ -119,7 +125,7 @@ func (d *Commento) convert(r io.Reader, siteID string) (ch chan store.Comment) {
User: u,
Text: comment.Markdown,
Timestamp: comment.CreationDate,
ParentID: comment.ParentHex,
ParentID: parentID,
Imported: true,
}

Expand Down
152 changes: 152 additions & 0 deletions backend/app/rest/api/migrator_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"mime/multipart"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"time"
Expand Down Expand Up @@ -49,6 +50,22 @@ func TestMigrator_Import(t *testing.T) {
assert.NoError(t, resp.Body.Close())

waitForMigrationCompletion(t, ts)

res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
require.Equal(t, http.StatusOK, code)
comments := commentsWithInfo{}
err = json.Unmarshal([]byte(res), &comments)
require.NoError(t, err)
require.Equal(t, 1, comments.Info.Count)
require.Equal(t, 1, len(comments.Comments))

res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://radio-t.com/blah1")
require.Equal(t, http.StatusOK, code)
comments = commentsWithInfo{}
err = json.Unmarshal([]byte(res), &comments)
require.NoError(t, err)
require.Equal(t, 1, comments.Info.Count)
require.Equal(t, 1, len(comments.Comments))
}

func TestMigrator_ImportForm(t *testing.T) {
Expand Down Expand Up @@ -84,6 +101,22 @@ func TestMigrator_ImportForm(t *testing.T) {
assert.NoError(t, resp.Body.Close())

waitForMigrationCompletion(t, ts)

res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
require.Equal(t, http.StatusOK, code)
comments := commentsWithInfo{}
err = json.Unmarshal([]byte(res), &comments)
require.NoError(t, err)
require.Equal(t, 1, comments.Info.Count)
require.Equal(t, 1, len(comments.Comments))

res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://radio-t.com/blah1")
require.Equal(t, http.StatusOK, code)
comments = commentsWithInfo{}
err = json.Unmarshal([]byte(res), &comments)
require.NoError(t, err)
require.Equal(t, 1, comments.Info.Count)
require.Equal(t, 1, len(comments.Comments))
}

func TestMigrator_ImportFromWP(t *testing.T) {
Expand All @@ -108,6 +141,22 @@ func TestMigrator_ImportFromWP(t *testing.T) {
assert.NoError(t, resp.Body.Close())

waitForMigrationCompletion(t, ts)

res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://realmenweardress.es/2010/07/do-you-rp/")
require.Equal(t, http.StatusOK, code)
comments := commentsWithInfo{}
err = json.Unmarshal([]byte(res), &comments)
require.NoError(t, err)
require.Equal(t, 3, comments.Info.Count)
require.Equal(t, 3, len(comments.Comments))

res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://realmenweardress.es/2010/07/do-you-rp/")
require.Equal(t, http.StatusOK, code)
comments = commentsWithInfo{}
err = json.Unmarshal([]byte(res), &comments)
require.NoError(t, err)
require.Equal(t, 3, comments.Info.Count)
require.Equal(t, 2, len(comments.Comments), "2 comments with 1 reply")
}

func TestMigrator_ImportFromCommento(t *testing.T) {
Expand Down Expand Up @@ -137,6 +186,63 @@ func TestMigrator_ImportFromCommento(t *testing.T) {
assert.NoError(t, resp.Body.Close())

waitForMigrationCompletion(t, ts)

res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://example.com/blog/post/1")
require.Equal(t, http.StatusOK, code)
comments := commentsWithInfo{}
err = json.Unmarshal([]byte(res), &comments)
require.NoError(t, err)
require.Equal(t, 1, comments.Info.Count)
require.Equal(t, 1, len(comments.Comments))

res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://example.com/blog/post/1")
require.Equal(t, http.StatusOK, code)
comments = commentsWithInfo{}
err = json.Unmarshal([]byte(res), &comments)
require.NoError(t, err)
require.Equal(t, 1, comments.Info.Count)
require.Equal(t, 1, len(comments.Comments))
}

func TestMigrator_ImportFromCommentoJSON(t *testing.T) {
ts, _, teardown := startupT(t)
defer teardown()

r, err := os.Open("testdata/commento.json")
require.NoError(t, err)

client := &http.Client{Timeout: 1 * time.Second}
defer client.CloseIdleConnections()
req, err := http.NewRequest("POST", ts.URL+"/api/v1/admin/import?site=remark42&provider=commento", r)
assert.NoError(t, err)
req.Header.Add("Content-Type", "application/json; charset=utf-8")
req.SetBasicAuth("admin", "password")
resp, err := client.Do(req)
assert.NoError(t, err)
assert.Equal(t, http.StatusAccepted, resp.StatusCode)

b, err := io.ReadAll(resp.Body)
assert.NoError(t, err)
assert.Equal(t, "{\"status\":\"import request accepted\"}\n", string(b))
assert.NoError(t, resp.Body.Close())

waitForMigrationCompletion(t, ts)

res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://example.com/example")
require.Equal(t, http.StatusOK, code)
comments := commentsWithInfo{}
err = json.Unmarshal([]byte(res), &comments)
require.NoError(t, err)
require.Equal(t, 7, comments.Info.Count)
require.Equal(t, 7, len(comments.Comments))

res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://example.com/example")
require.Equal(t, http.StatusOK, code)
comments = commentsWithInfo{}
err = json.Unmarshal([]byte(res), &comments)
require.NoError(t, err)
require.Equal(t, 7, comments.Info.Count)
require.Equal(t, 5, len(comments.Comments), "five comments with two replies")
}

func TestMigrator_ImportRejected(t *testing.T) {
Expand Down Expand Up @@ -197,6 +303,20 @@ func TestMigrator_ImportDouble(t *testing.T) {
assert.NoError(t, resp.Body.Close())
assert.Equal(t, http.StatusConflict, resp.StatusCode)
waitForMigrationCompletion(t, ts)

res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
require.Equal(t, http.StatusOK, code)
comments := commentsWithInfo{}
err = json.Unmarshal([]byte(res), &comments)
require.NoError(t, err)
require.Equal(t, 50, comments.Info.Count)

res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://radio-t.com/blah1")
require.Equal(t, http.StatusOK, code)
comments = commentsWithInfo{}
err = json.Unmarshal([]byte(res), &comments)
require.NoError(t, err)
require.Equal(t, 50, comments.Info.Count)
}

func TestMigrator_ImportWaitExpired(t *testing.T) {
Expand Down Expand Up @@ -236,6 +356,14 @@ func TestMigrator_ImportWaitExpired(t *testing.T) {
assert.Equal(t, http.StatusGatewayTimeout, resp.StatusCode)

waitForMigrationCompletion(t, ts)

res, code := get(t, ts.URL+"/api/v1/find?site=remark42&url=https://example.com/example")
require.Equal(t, http.StatusOK, code)
comments := commentsWithInfo{}
err = json.Unmarshal([]byte(res), &comments)
require.NoError(t, err)
require.Equal(t, 0, comments.Info.Count)
require.Equal(t, 0, len(comments.Comments))
}

func TestMigrator_Export(t *testing.T) {
Expand Down Expand Up @@ -339,6 +467,7 @@ func TestMigrator_Remap(t *testing.T) {
err = json.Unmarshal([]byte(res), &comments)
require.NoError(t, err)
require.Equal(t, 2, comments.Info.Count)
require.Equal(t, 2, len(comments.Comments))
require.False(t, comments.Info.ReadOnly)

res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/")
Expand All @@ -347,6 +476,7 @@ func TestMigrator_Remap(t *testing.T) {
err = json.Unmarshal([]byte(res), &comments)
require.NoError(t, err)
require.Equal(t, 1, comments.Info.Count)
require.Equal(t, 1, len(comments.Comments))
require.True(t, comments.Info.ReadOnly)

// we want remap urls to another domain - www.remark42.com
Expand All @@ -364,6 +494,16 @@ func TestMigrator_Remap(t *testing.T) {
err = json.Unmarshal([]byte(res), &comments)
require.NoError(t, err)
require.Equal(t, 2, comments.Info.Count)
require.Equal(t, 2, len(comments.Comments))
require.False(t, comments.Info.ReadOnly)

res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://www.remark42.com/demo/")
require.Equal(t, http.StatusOK, code)
comments = commentsWithInfo{}
err = json.Unmarshal([]byte(res), &comments)
require.NoError(t, err)
require.Equal(t, 2, comments.Info.Count)
require.Equal(t, 2, len(comments.Comments))
require.False(t, comments.Info.ReadOnly)

res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://www.remark42.com/demo-another/")
Expand All @@ -372,6 +512,16 @@ func TestMigrator_Remap(t *testing.T) {
err = json.Unmarshal([]byte(res), &comments)
require.NoError(t, err)
require.Equal(t, 1, comments.Info.Count)
require.Equal(t, 1, len(comments.Comments))
require.True(t, comments.Info.ReadOnly)

res, code = get(t, ts.URL+"/api/v1/find?site=remark42&format=tree&url=https://www.remark42.com/demo-another/")
require.Equal(t, http.StatusOK, code)
comments = commentsWithInfo{}
err = json.Unmarshal([]byte(res), &comments)
require.NoError(t, err)
require.Equal(t, 1, comments.Info.Count)
require.Equal(t, 1, len(comments.Comments))
require.True(t, comments.Info.ReadOnly)

// should find nothing from previous url
Expand All @@ -381,13 +531,15 @@ func TestMigrator_Remap(t *testing.T) {
err = json.Unmarshal([]byte(res), &comments)
require.NoError(t, err)
require.Equal(t, 0, comments.Info.Count)
require.Equal(t, 0, len(comments.Comments))

res, code = get(t, ts.URL+"/api/v1/find?site=remark42&url=https://remark42.com/demo-another/")
require.Equal(t, http.StatusOK, code)
comments = commentsWithInfo{}
err = json.Unmarshal([]byte(res), &comments)
require.NoError(t, err)
require.Equal(t, 0, comments.Info.Count)
require.Equal(t, 0, len(comments.Comments))
}

func TestMigrator_RemapReject(t *testing.T) {
Expand Down
116 changes: 116 additions & 0 deletions backend/app/rest/api/testdata/commento.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
{
"version": 1,
"comments": [
{
"commentHex": "e7a2ef4b4aa1414a7ee65a989889aaecd9d5e7e3bca598ea7a967b4dbcaa8e11",
"domain": "example.com",
"url": "https://example.com/example",
"commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
"markdown": "",
"html": "",
"parentHex": "root",
"score": 0,
"state": "approved",
"creationDate": "2022-10-25T07:25:46.807555Z",
"direction": 0,
"deleted": false
},
{
"commentHex": "a29e741145daceb4ca5b3e5e279e05b56f73c04703d93b944718ef757e15317f",
"domain": "example.com",
"url": "https://example.com/example",
"commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
"markdown": "",
"html": "",
"parentHex": "root",
"score": 0,
"state": "approved",
"creationDate": "2023-07-26T12:24:55.058552Z",
"direction": 0,
"deleted": false
},
{
"commentHex": "46baf36433830a4e8bda1de56290cf5fd74c08bfa844fee4ec1744985dc77010",
"domain": "example.com",
"url": "https://example.com/example",
"commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
"markdown": "",
"html": "",
"parentHex": "root",
"score": 0,
"state": "approved",
"creationDate": "2023-10-31T11:03:25.403282Z",
"direction": 0,
"deleted": false
},
{
"commentHex": "6d3bb64ff73b5f9d6a959212ffde472a51abf8bdefaa5ed843659796bceef9de",
"domain": "example.com",
"url": "https://example.com/example",
"commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
"markdown": "",
"html": "",
"parentHex": "46baf36433830a4e8bda1de56290cf5fd74c08bfa844fee4ec1744985dc77010",
"score": 0,
"state": "approved",
"creationDate": "2023-11-01T22:23:47.112062Z",
"direction": 0,
"deleted": false
},
{
"commentHex": "23fcfcd03745ed71a9d23a9b59387a313df57e5c0faad8ba5dc96112766312c5",
"domain": "example.com",
"url": "https://example.com/example",
"commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
"markdown": "",
"html": "",
"parentHex": "root",
"score": 0,
"state": "approved",
"creationDate": "2023-10-23T12:33:03.370182Z",
"direction": 0,
"deleted": false
},
{
"commentHex": "d0ad6f11cf0c5f8e17457a378a6bb789f412c6b7ef7ada4ae06ec8451f7a18aa",
"domain": "example.com",
"url": "https://example.com/example",
"commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
"markdown": "",
"html": "",
"parentHex": "root",
"score": 0,
"state": "approved",
"creationDate": "2023-10-18T01:18:38.193625Z",
"direction": 0,
"deleted": false
},
{
"commentHex": "098960fd01c1fc7c0d3ea428f52fab97ea5c18aa52f3565bba679224daddc687",
"domain": "example.com",
"url": "https://example.com/example",
"commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
"markdown": "",
"html": "",
"parentHex": "23fcfcd03745ed71a9d23a9b59387a313df57e5c0faad8ba5dc96112766312c5",
"score": 0,
"state": "approved",
"creationDate": "2023-11-01T22:24:04.639965Z",
"direction": 0,
"deleted": false
}
],
"commenters": [
{
"commenterHex": "018407e4b12b35f43b1d804d82607b341bef80c4325dd047d93f2cbb439cff85",
"email": "undefined",
"name": "blank",
"link": "undefined",
"photo": "undefined",
"provider": "anon",
"joinDate": "2022-06-09T15:54:29.865919Z",
"isModerator": false,
"deleted": false
}
]
}

0 comments on commit ce678bf

Please sign in to comment.